{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "54159c1d-35a7-4aea-8280-49c7f2414a2b",
   "metadata": {},
   "source": [
    "# MathQA, Self-Consistency"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "id": "7a8b3c22-74cd-47b4-9ecf-28c36516a951",
   "metadata": {},
   "outputs": [],
   "source": [
    "import openai\n",
    "import re\n",
    "import json\n",
    "import spacy\n",
    "import time\n",
    "\n",
    "import numpy as np\n",
    "\n",
    "from tqdm import tqdm\n",
    "from collections import Counter"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 135,
   "id": "92aa00f0-5a07-41de-9462-c1e2b7640a79",
   "metadata": {},
   "outputs": [],
   "source": [
    "train_data = json.load(open('MathQA/train.json'))\n",
    "\n",
    "test_idx = np.load('MathQA/test_idx.npy')\n",
    "test_data_ = json.load(open('MathQA/test.json'))\n",
    "test_data = []\n",
    "for i in test_idx: test_data.append(test_data_[i])\n",
    "\n",
    "dev_200_idx = np.load('MathQA/dev_200_idx.npy')\n",
    "dev_data = json.load(open('MathQA/dev.json'))\n",
    "dev_data_200 = []\n",
    "for i in dev_200_idx: dev_data_200.append(dev_data[i])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 128,
   "id": "a52f1952-5b28-4ab7-a7aa-3ff68b381f66",
   "metadata": {},
   "outputs": [],
   "source": [
    "prompt_complex = open('MathQA/prompt_complex.txt').read()\n",
    "prompt_long_formula = open('MathQA/prompt_long_formula.txt').read()\n",
    "prompt_original = open('MathQA/prompt_original.txt').read()\n",
    "prompt_random = open('MathQA/prompt_random.txt').read()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "45d3f33c-a684-41f1-a0a1-c222d5f0c427",
   "metadata": {},
   "source": [
    "# Preparations"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 150,
   "id": "9a89412a-7bb6-4bb5-b361-d8722b027910",
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_ans_dict(c):\n",
    "    if(c.startswith(\"['\")):\n",
    "        c_ = eval(c)\n",
    "        ans_dict = {}\n",
    "        for ac in c_:\n",
    "            idx_, num_ = ac.split(' ) ')[-2:]\n",
    "            if('a' in idx_): ans_dict['a'] = num_\n",
    "            if('b' in idx_): ans_dict['b'] = num_\n",
    "            if('c' in idx_): ans_dict['c'] = num_\n",
    "            if('d' in idx_): ans_dict['d'] = num_\n",
    "            if('e' in idx_): ans_dict['e'] = num_\n",
    "    else:\n",
    "        ans_dict = {}\n",
    "        pattern = ' , [a-e] \\) '\n",
    "        c_ = re.split(pattern, c)\n",
    "        c_[0] = c_[0].split(' ) ')[-1]\n",
    "        for i, c in zip(['a', 'b', 'c', 'd', 'e'], c_):\n",
    "            ans_dict[i] = c\n",
    "    return ans_dict\n",
    "\n",
    "def find_answer(pred_str):\n",
    "    pattern = '\\d*\\.?\\d+'\n",
    "    pred = re.findall(pattern, pred_str)\n",
    "    if(len(pred) >= 1): pred = pred[-1]\n",
    "    else: pred = None\n",
    "    return pred\n",
    "\n",
    "def find_majority(ans_strs, return_counter=False):\n",
    "    ans = []\n",
    "    for s in ans_strs:\n",
    "        pred = find_answer(s)\n",
    "        ans.append(pred)\n",
    "    ans = Counter(ans)\n",
    "    # print(ans)\n",
    "    major = ans.most_common(1)[0][0]\n",
    "    i = 1\n",
    "    while major is None:\n",
    "        major = ans.most_common(i + 1)[i][0]\n",
    "    if(return_counter): return major, ans\n",
    "    else: return major\n",
    "\n",
    "def test_answer(pred, ans_str):\n",
    "    pattern = '\\d*\\.?\\d+'\n",
    "    gold = re.findall(pattern, ans_str)\n",
    "    gold = gold[-1]\n",
    "    return pred == gold"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 122,
   "id": "630b0623-445e-4299-bd4c-630cdb0c0ab9",
   "metadata": {},
   "outputs": [],
   "source": [
    "def majority_over_hard(answer_samples, K, use_easy=False):\n",
    "    \"\"\"majority voting over hard / easy sample\n",
    "    \"\"\"\n",
    "    sample_answer_lens = []\n",
    "    ans_pred = []\n",
    "    for a in answer_samples:\n",
    "        a_len = len(a)\n",
    "        sample_answer_lens.append(a_len)\n",
    "        \n",
    "        ans = find_answer(a[-1])\n",
    "        ans_pred.append(ans)\n",
    "        \n",
    "    sample_answer_lens = np.array(sample_answer_lens)\n",
    "    if(use_easy):\n",
    "        top_index = np.argsort(sample_answer_lens)[:K]\n",
    "    else:\n",
    "        top_index = np.argsort(sample_answer_lens)[::-1][:K]\n",
    "    top_lens = [sample_answer_lens[i] for i in top_index]\n",
    "    \n",
    "    ans_candidates = [ans_pred[i] for i in top_index]\n",
    "    \n",
    "    ans_candidates = Counter(ans_candidates)\n",
    "    major = ans_candidates.most_common(1)[0][0]\n",
    "    return top_index, top_lens, ans_candidates, major\n",
    "\n",
    "def majority_over_hard_and_long(answer_samples, K, short_step=False):\n",
    "    \"\"\"majority voting over hard / easy sample\n",
    "    \"\"\"\n",
    "    sample_answer_lens = {}\n",
    "    ans_pred = []\n",
    "    for a in answer_samples:\n",
    "        a_len = len(a)\n",
    "        if(a_len in sample_answer_lens): sample_answer_lens[a_len].append((a, sum(len(ai) for ai in a)))\n",
    "        else: sample_answer_lens[a_len] = [(a, sum(len(ai) for ai in a))]\n",
    "\n",
    "    for a_len in sample_answer_lens: \n",
    "        if(short_step): sample_answer_lens[a_len].sort(key = lambda x: x[1])\n",
    "        else: sample_answer_lens[a_len].sort(key = lambda x: x[1], reverse=True)\n",
    "    all_lens = list(sample_answer_lens.keys())\n",
    "    all_lens.sort(reverse=True)\n",
    "    \n",
    "    ans_candidates = []\n",
    "    reached_k = False\n",
    "    for a_len in all_lens:\n",
    "        if(reached_k): break\n",
    "        for a in sample_answer_lens[a_len]:\n",
    "            ans_candidates.append(find_answer(a[0][-1]))\n",
    "            if(len(ans_candidates) == K): \n",
    "                reached_k = True\n",
    "                break\n",
    "    \n",
    "    ans_candidates = Counter(ans_candidates)\n",
    "    major = ans_candidates.most_common(1)[0][0]\n",
    "    return sample_answer_lens, ans_candidates, major\n",
    "\n",
    "def read_sample_answer(filename):\n",
    "    questions = []\n",
    "    answers = []\n",
    "    answer_reasons = []\n",
    "    answer_labels = [] \n",
    "\n",
    "    with open(filename) as fd:\n",
    "        lines = fd.readlines()\n",
    "        mode = 'none'\n",
    "        for l in lines:\n",
    "            if(l.startswith('Q: ')):\n",
    "                q = l[:-1]\n",
    "                questions.append(q)\n",
    "                ans = []\n",
    "            elif(l.startswith('A_gold')):\n",
    "                answers.append(ans)\n",
    "                mode = 'seek_answer'\n",
    "                ans_gold = []\n",
    "            elif(l.startswith(\"A_model:\")):\n",
    "                mode = 'seek_sample'\n",
    "                ans_sample = []\n",
    "            else:\n",
    "                if(mode == 'seek_sample'):\n",
    "                    if(l == '\\n'): continue\n",
    "                    ans_sample.append(l)\n",
    "                    if(l.endswith('[EOA]\\n')):\n",
    "                        ans.append(ans_sample)\n",
    "                        # mode = 'none'\n",
    "                        ans_sample = []\n",
    "                elif(mode == 'seek_answer'):\n",
    "                    # if(l.startswith('####')):\n",
    "                    #     answer_reasons.append(ans_gold)\n",
    "                    #     answer_labels.append(l.split()[-1])\n",
    "                    #     mode = 'none'\n",
    "                    # else:\n",
    "                    answer_reasons.append(l)\n",
    "                    answer_labels.append(find_answer(l))\n",
    "                    mode = 'none'\n",
    "    return questions, answers, answer_reasons, answer_labels\n",
    "                        \n",
    "def test_voting_over_k(K, answers, answer_labels, vote_over_easy=False):\n",
    "    total_case = 0\n",
    "    acc = 0\n",
    "    for a, p in zip(answers, answer_labels):\n",
    "        top_index, top_lens, ans_candidates, major = majority_over_hard(a, K, vote_over_easy)\n",
    "        if(major == p): acc += 1\n",
    "        total_case += 1\n",
    "\n",
    "    print('%d total %d correct acc %.4f' % (total_case, acc, float(acc) / total_case))\n",
    "    return \n",
    "\n",
    "def test_voting_over_hard_and_long(K, answers, answer_labels):\n",
    "    total_case = 0\n",
    "    acc = 0\n",
    "    for a, p in zip(answers, answer_labels):\n",
    "        sample_answer_lens, ans_candidates, major = majority_over_hard_and_long(a, K)\n",
    "        if(major == p): acc += 1\n",
    "        total_case += 1\n",
    "\n",
    "    print('%d total %d correct acc %.4f' % (total_case, acc, float(acc) / total_case))\n",
    "    return "
   ]
  },
  {
   "cell_type": "markdown",
   "id": "95b7563b-236a-4c4f-bee8-14e20dcba310",
   "metadata": {},
   "source": [
    "# Complex Prompt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "id": "4a327a9b-c464-42c2-871e-ce9ebc6549bf",
   "metadata": {
    "collapsed": true,
    "jupyter": {
     "outputs_hidden": true
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 49%|████████████████████████████████████████████████████████████████▏                                                                  | 98/200 [44:18<46:07, 27.13s/it]\n"
     ]
    },
    {
     "ename": "APIConnectionError",
     "evalue": "Error communicating with OpenAI",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mtimeout\u001b[0m                                   Traceback (most recent call last)",
      "File \u001b[0;32m~/miniconda3/envs/dl_py3.8/lib/python3.8/site-packages/urllib3/connectionpool.py:449\u001b[0m, in \u001b[0;36mHTTPConnectionPool._make_request\u001b[0;34m(self, conn, method, url, timeout, chunked, **httplib_request_kw)\u001b[0m\n\u001b[1;32m    445\u001b[0m         \u001b[38;5;28;01mexcept\u001b[39;00m \u001b[38;5;167;01mBaseException\u001b[39;00m \u001b[38;5;28;01mas\u001b[39;00m e:\n\u001b[1;32m    446\u001b[0m             \u001b[38;5;66;03m# Remove the TypeError from the exception chain in\u001b[39;00m\n\u001b[1;32m    447\u001b[0m             \u001b[38;5;66;03m# Python 3 (including for exceptions like SystemExit).\u001b[39;00m\n\u001b[1;32m    448\u001b[0m             \u001b[38;5;66;03m# Otherwise it looks like a bug in the code.\u001b[39;00m\n\u001b[0;32m--> 449\u001b[0m             \u001b[43msix\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mraise_from\u001b[49m\u001b[43m(\u001b[49m\u001b[43me\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;28;43;01mNone\u001b[39;49;00m\u001b[43m)\u001b[49m\n\u001b[1;32m    450\u001b[0m \u001b[38;5;28;01mexcept\u001b[39;00m (SocketTimeout, BaseSSLError, SocketError) \u001b[38;5;28;01mas\u001b[39;00m e:\n",
      "File \u001b[0;32m<string>:3\u001b[0m, in \u001b[0;36mraise_from\u001b[0;34m(value, from_value)\u001b[0m\n",
      "File \u001b[0;32m~/miniconda3/envs/dl_py3.8/lib/python3.8/site-packages/urllib3/connectionpool.py:444\u001b[0m, in \u001b[0;36mHTTPConnectionPool._make_request\u001b[0;34m(self, conn, method, url, timeout, chunked, **httplib_request_kw)\u001b[0m\n\u001b[1;32m    443\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[0;32m--> 444\u001b[0m     httplib_response \u001b[38;5;241m=\u001b[39m \u001b[43mconn\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mgetresponse\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    445\u001b[0m \u001b[38;5;28;01mexcept\u001b[39;00m \u001b[38;5;167;01mBaseException\u001b[39;00m \u001b[38;5;28;01mas\u001b[39;00m e:\n\u001b[1;32m    446\u001b[0m     \u001b[38;5;66;03m# Remove the TypeError from the exception chain in\u001b[39;00m\n\u001b[1;32m    447\u001b[0m     \u001b[38;5;66;03m# Python 3 (including for exceptions like SystemExit).\u001b[39;00m\n\u001b[1;32m    448\u001b[0m     \u001b[38;5;66;03m# Otherwise it looks like a bug in the code.\u001b[39;00m\n",
      "File \u001b[0;32m~/miniconda3/envs/dl_py3.8/lib/python3.8/http/client.py:1348\u001b[0m, in \u001b[0;36mHTTPConnection.getresponse\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m   1347\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[0;32m-> 1348\u001b[0m     \u001b[43mresponse\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mbegin\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m   1349\u001b[0m \u001b[38;5;28;01mexcept\u001b[39;00m \u001b[38;5;167;01mConnectionError\u001b[39;00m:\n",
      "File \u001b[0;32m~/miniconda3/envs/dl_py3.8/lib/python3.8/http/client.py:316\u001b[0m, in \u001b[0;36mHTTPResponse.begin\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m    315\u001b[0m \u001b[38;5;28;01mwhile\u001b[39;00m \u001b[38;5;28;01mTrue\u001b[39;00m:\n\u001b[0;32m--> 316\u001b[0m     version, status, reason \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_read_status\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    317\u001b[0m     \u001b[38;5;28;01mif\u001b[39;00m status \u001b[38;5;241m!=\u001b[39m CONTINUE:\n",
      "File \u001b[0;32m~/miniconda3/envs/dl_py3.8/lib/python3.8/http/client.py:277\u001b[0m, in \u001b[0;36mHTTPResponse._read_status\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m    276\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21m_read_status\u001b[39m(\u001b[38;5;28mself\u001b[39m):\n\u001b[0;32m--> 277\u001b[0m     line \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mstr\u001b[39m(\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mfp\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mreadline\u001b[49m\u001b[43m(\u001b[49m\u001b[43m_MAXLINE\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m+\u001b[39;49m\u001b[43m \u001b[49m\u001b[38;5;241;43m1\u001b[39;49m\u001b[43m)\u001b[49m, \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124miso-8859-1\u001b[39m\u001b[38;5;124m\"\u001b[39m)\n\u001b[1;32m    278\u001b[0m     \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mlen\u001b[39m(line) \u001b[38;5;241m>\u001b[39m _MAXLINE:\n",
      "File \u001b[0;32m~/miniconda3/envs/dl_py3.8/lib/python3.8/socket.py:669\u001b[0m, in \u001b[0;36mSocketIO.readinto\u001b[0;34m(self, b)\u001b[0m\n\u001b[1;32m    668\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[0;32m--> 669\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_sock\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mrecv_into\u001b[49m\u001b[43m(\u001b[49m\u001b[43mb\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    670\u001b[0m \u001b[38;5;28;01mexcept\u001b[39;00m timeout:\n",
      "File \u001b[0;32m~/miniconda3/envs/dl_py3.8/lib/python3.8/ssl.py:1241\u001b[0m, in \u001b[0;36mSSLSocket.recv_into\u001b[0;34m(self, buffer, nbytes, flags)\u001b[0m\n\u001b[1;32m   1238\u001b[0m         \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;167;01mValueError\u001b[39;00m(\n\u001b[1;32m   1239\u001b[0m           \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mnon-zero flags not allowed in calls to recv_into() on \u001b[39m\u001b[38;5;132;01m%s\u001b[39;00m\u001b[38;5;124m\"\u001b[39m \u001b[38;5;241m%\u001b[39m\n\u001b[1;32m   1240\u001b[0m           \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m\u001b[38;5;18m__class__\u001b[39m)\n\u001b[0;32m-> 1241\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mread\u001b[49m\u001b[43m(\u001b[49m\u001b[43mnbytes\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mbuffer\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m   1242\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n",
      "File \u001b[0;32m~/miniconda3/envs/dl_py3.8/lib/python3.8/ssl.py:1099\u001b[0m, in \u001b[0;36mSSLSocket.read\u001b[0;34m(self, len, buffer)\u001b[0m\n\u001b[1;32m   1098\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m buffer \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m:\n\u001b[0;32m-> 1099\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_sslobj\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mread\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;28;43mlen\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mbuffer\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m   1100\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n",
      "\u001b[0;31mtimeout\u001b[0m: The read operation timed out",
      "\nDuring handling of the above exception, another exception occurred:\n",
      "\u001b[0;31mReadTimeoutError\u001b[0m                          Traceback (most recent call last)",
      "File \u001b[0;32m~/miniconda3/envs/dl_py3.8/lib/python3.8/site-packages/requests/adapters.py:489\u001b[0m, in \u001b[0;36mHTTPAdapter.send\u001b[0;34m(self, request, stream, timeout, verify, cert, proxies)\u001b[0m\n\u001b[1;32m    488\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m chunked:\n\u001b[0;32m--> 489\u001b[0m     resp \u001b[38;5;241m=\u001b[39m \u001b[43mconn\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43murlopen\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m    490\u001b[0m \u001b[43m        \u001b[49m\u001b[43mmethod\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mrequest\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mmethod\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    491\u001b[0m \u001b[43m        \u001b[49m\u001b[43murl\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43murl\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    492\u001b[0m \u001b[43m        \u001b[49m\u001b[43mbody\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mrequest\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mbody\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    493\u001b[0m \u001b[43m        \u001b[49m\u001b[43mheaders\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mrequest\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mheaders\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    494\u001b[0m \u001b[43m        \u001b[49m\u001b[43mredirect\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;28;43;01mFalse\u001b[39;49;00m\u001b[43m,\u001b[49m\n\u001b[1;32m    495\u001b[0m \u001b[43m        \u001b[49m\u001b[43massert_same_host\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;28;43;01mFalse\u001b[39;49;00m\u001b[43m,\u001b[49m\n\u001b[1;32m    496\u001b[0m \u001b[43m        \u001b[49m\u001b[43mpreload_content\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;28;43;01mFalse\u001b[39;49;00m\u001b[43m,\u001b[49m\n\u001b[1;32m    497\u001b[0m \u001b[43m        \u001b[49m\u001b[43mdecode_content\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;28;43;01mFalse\u001b[39;49;00m\u001b[43m,\u001b[49m\n\u001b[1;32m    498\u001b[0m \u001b[43m        \u001b[49m\u001b[43mretries\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mmax_retries\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    499\u001b[0m \u001b[43m        \u001b[49m\u001b[43mtimeout\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mtimeout\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    500\u001b[0m \u001b[43m    \u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    502\u001b[0m \u001b[38;5;66;03m# Send the request.\u001b[39;00m\n\u001b[1;32m    503\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n",
      "File \u001b[0;32m~/miniconda3/envs/dl_py3.8/lib/python3.8/site-packages/urllib3/connectionpool.py:787\u001b[0m, in \u001b[0;36mHTTPConnectionPool.urlopen\u001b[0;34m(self, method, url, body, headers, retries, redirect, assert_same_host, timeout, pool_timeout, release_conn, chunked, body_pos, **response_kw)\u001b[0m\n\u001b[1;32m    785\u001b[0m     e \u001b[38;5;241m=\u001b[39m ProtocolError(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mConnection aborted.\u001b[39m\u001b[38;5;124m\"\u001b[39m, e)\n\u001b[0;32m--> 787\u001b[0m retries \u001b[38;5;241m=\u001b[39m \u001b[43mretries\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mincrement\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m    788\u001b[0m \u001b[43m    \u001b[49m\u001b[43mmethod\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43murl\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43merror\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43me\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43m_pool\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43m_stacktrace\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43msys\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mexc_info\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\u001b[43m[\u001b[49m\u001b[38;5;241;43m2\u001b[39;49m\u001b[43m]\u001b[49m\n\u001b[1;32m    789\u001b[0m \u001b[43m\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    790\u001b[0m retries\u001b[38;5;241m.\u001b[39msleep()\n",
      "File \u001b[0;32m~/miniconda3/envs/dl_py3.8/lib/python3.8/site-packages/urllib3/util/retry.py:550\u001b[0m, in \u001b[0;36mRetry.increment\u001b[0;34m(self, method, url, response, error, _pool, _stacktrace)\u001b[0m\n\u001b[1;32m    549\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m read \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;28;01mFalse\u001b[39;00m \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_is_method_retryable(method):\n\u001b[0;32m--> 550\u001b[0m     \u001b[38;5;28;01mraise\u001b[39;00m \u001b[43msix\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mreraise\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;28;43mtype\u001b[39;49m\u001b[43m(\u001b[49m\u001b[43merror\u001b[49m\u001b[43m)\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43merror\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43m_stacktrace\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    551\u001b[0m \u001b[38;5;28;01melif\u001b[39;00m read \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m:\n",
      "File \u001b[0;32m~/miniconda3/envs/dl_py3.8/lib/python3.8/site-packages/urllib3/packages/six.py:770\u001b[0m, in \u001b[0;36mreraise\u001b[0;34m(tp, value, tb)\u001b[0m\n\u001b[1;32m    769\u001b[0m         \u001b[38;5;28;01mraise\u001b[39;00m value\u001b[38;5;241m.\u001b[39mwith_traceback(tb)\n\u001b[0;32m--> 770\u001b[0m     \u001b[38;5;28;01mraise\u001b[39;00m value\n\u001b[1;32m    771\u001b[0m \u001b[38;5;28;01mfinally\u001b[39;00m:\n",
      "File \u001b[0;32m~/miniconda3/envs/dl_py3.8/lib/python3.8/site-packages/urllib3/connectionpool.py:703\u001b[0m, in \u001b[0;36mHTTPConnectionPool.urlopen\u001b[0;34m(self, method, url, body, headers, retries, redirect, assert_same_host, timeout, pool_timeout, release_conn, chunked, body_pos, **response_kw)\u001b[0m\n\u001b[1;32m    702\u001b[0m \u001b[38;5;66;03m# Make the request on the httplib connection object.\u001b[39;00m\n\u001b[0;32m--> 703\u001b[0m httplib_response \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_make_request\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m    704\u001b[0m \u001b[43m    \u001b[49m\u001b[43mconn\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    705\u001b[0m \u001b[43m    \u001b[49m\u001b[43mmethod\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    706\u001b[0m \u001b[43m    \u001b[49m\u001b[43murl\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    707\u001b[0m \u001b[43m    \u001b[49m\u001b[43mtimeout\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mtimeout_obj\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    708\u001b[0m \u001b[43m    \u001b[49m\u001b[43mbody\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mbody\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    709\u001b[0m \u001b[43m    \u001b[49m\u001b[43mheaders\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mheaders\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    710\u001b[0m \u001b[43m    \u001b[49m\u001b[43mchunked\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mchunked\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    711\u001b[0m \u001b[43m\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    713\u001b[0m \u001b[38;5;66;03m# If we're going to release the connection in ``finally:``, then\u001b[39;00m\n\u001b[1;32m    714\u001b[0m \u001b[38;5;66;03m# the response doesn't need to know about the connection. Otherwise\u001b[39;00m\n\u001b[1;32m    715\u001b[0m \u001b[38;5;66;03m# it will also try to release it and we'll have a double-release\u001b[39;00m\n\u001b[1;32m    716\u001b[0m \u001b[38;5;66;03m# mess.\u001b[39;00m\n",
      "File \u001b[0;32m~/miniconda3/envs/dl_py3.8/lib/python3.8/site-packages/urllib3/connectionpool.py:451\u001b[0m, in \u001b[0;36mHTTPConnectionPool._make_request\u001b[0;34m(self, conn, method, url, timeout, chunked, **httplib_request_kw)\u001b[0m\n\u001b[1;32m    450\u001b[0m \u001b[38;5;28;01mexcept\u001b[39;00m (SocketTimeout, BaseSSLError, SocketError) \u001b[38;5;28;01mas\u001b[39;00m e:\n\u001b[0;32m--> 451\u001b[0m     \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_raise_timeout\u001b[49m\u001b[43m(\u001b[49m\u001b[43merr\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43me\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43murl\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43murl\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mtimeout_value\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mread_timeout\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    452\u001b[0m     \u001b[38;5;28;01mraise\u001b[39;00m\n",
      "File \u001b[0;32m~/miniconda3/envs/dl_py3.8/lib/python3.8/site-packages/urllib3/connectionpool.py:340\u001b[0m, in \u001b[0;36mHTTPConnectionPool._raise_timeout\u001b[0;34m(self, err, url, timeout_value)\u001b[0m\n\u001b[1;32m    339\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28misinstance\u001b[39m(err, SocketTimeout):\n\u001b[0;32m--> 340\u001b[0m     \u001b[38;5;28;01mraise\u001b[39;00m ReadTimeoutError(\n\u001b[1;32m    341\u001b[0m         \u001b[38;5;28mself\u001b[39m, url, \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mRead timed out. (read timeout=\u001b[39m\u001b[38;5;132;01m%s\u001b[39;00m\u001b[38;5;124m)\u001b[39m\u001b[38;5;124m\"\u001b[39m \u001b[38;5;241m%\u001b[39m timeout_value\n\u001b[1;32m    342\u001b[0m     )\n\u001b[1;32m    344\u001b[0m \u001b[38;5;66;03m# See the above comment about EAGAIN in Python 3. In Python 2 we have\u001b[39;00m\n\u001b[1;32m    345\u001b[0m \u001b[38;5;66;03m# to specifically catch it and throw the timeout error\u001b[39;00m\n",
      "\u001b[0;31mReadTimeoutError\u001b[0m: HTTPSConnectionPool(host='api.openai.com', port=443): Read timed out. (read timeout=600)",
      "\nDuring handling of the above exception, another exception occurred:\n",
      "\u001b[0;31mReadTimeout\u001b[0m                               Traceback (most recent call last)",
      "File \u001b[0;32m~/miniconda3/envs/dl_py3.8/lib/python3.8/site-packages/openai/api_requestor.py:291\u001b[0m, in \u001b[0;36mAPIRequestor.request_raw\u001b[0;34m(self, method, url, params, supplied_headers, files, stream, request_id)\u001b[0m\n\u001b[1;32m    290\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[0;32m--> 291\u001b[0m     result \u001b[38;5;241m=\u001b[39m \u001b[43m_thread_context\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43msession\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mrequest\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m    292\u001b[0m \u001b[43m        \u001b[49m\u001b[43mmethod\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    293\u001b[0m \u001b[43m        \u001b[49m\u001b[43mabs_url\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    294\u001b[0m \u001b[43m        \u001b[49m\u001b[43mheaders\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mheaders\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    295\u001b[0m \u001b[43m        \u001b[49m\u001b[43mdata\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mdata\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    296\u001b[0m \u001b[43m        \u001b[49m\u001b[43mfiles\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mfiles\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    297\u001b[0m \u001b[43m        \u001b[49m\u001b[43mstream\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mstream\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    298\u001b[0m \u001b[43m        \u001b[49m\u001b[43mtimeout\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mTIMEOUT_SECS\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    299\u001b[0m \u001b[43m    \u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    300\u001b[0m \u001b[38;5;28;01mexcept\u001b[39;00m requests\u001b[38;5;241m.\u001b[39mexceptions\u001b[38;5;241m.\u001b[39mRequestException \u001b[38;5;28;01mas\u001b[39;00m e:\n",
      "File \u001b[0;32m~/miniconda3/envs/dl_py3.8/lib/python3.8/site-packages/requests/sessions.py:587\u001b[0m, in \u001b[0;36mSession.request\u001b[0;34m(self, method, url, params, data, headers, cookies, files, auth, timeout, allow_redirects, proxies, hooks, stream, verify, cert, json)\u001b[0m\n\u001b[1;32m    586\u001b[0m send_kwargs\u001b[38;5;241m.\u001b[39mupdate(settings)\n\u001b[0;32m--> 587\u001b[0m resp \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43msend\u001b[49m\u001b[43m(\u001b[49m\u001b[43mprep\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43msend_kwargs\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    589\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m resp\n",
      "File \u001b[0;32m~/miniconda3/envs/dl_py3.8/lib/python3.8/site-packages/requests/sessions.py:701\u001b[0m, in \u001b[0;36mSession.send\u001b[0;34m(self, request, **kwargs)\u001b[0m\n\u001b[1;32m    700\u001b[0m \u001b[38;5;66;03m# Send the request\u001b[39;00m\n\u001b[0;32m--> 701\u001b[0m r \u001b[38;5;241m=\u001b[39m \u001b[43madapter\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43msend\u001b[49m\u001b[43m(\u001b[49m\u001b[43mrequest\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    703\u001b[0m \u001b[38;5;66;03m# Total elapsed time of the request (approximately)\u001b[39;00m\n",
      "File \u001b[0;32m~/miniconda3/envs/dl_py3.8/lib/python3.8/site-packages/requests/adapters.py:578\u001b[0m, in \u001b[0;36mHTTPAdapter.send\u001b[0;34m(self, request, stream, timeout, verify, cert, proxies)\u001b[0m\n\u001b[1;32m    577\u001b[0m \u001b[38;5;28;01melif\u001b[39;00m \u001b[38;5;28misinstance\u001b[39m(e, ReadTimeoutError):\n\u001b[0;32m--> 578\u001b[0m     \u001b[38;5;28;01mraise\u001b[39;00m ReadTimeout(e, request\u001b[38;5;241m=\u001b[39mrequest)\n\u001b[1;32m    579\u001b[0m \u001b[38;5;28;01melif\u001b[39;00m \u001b[38;5;28misinstance\u001b[39m(e, _InvalidHeader):\n",
      "\u001b[0;31mReadTimeout\u001b[0m: HTTPSConnectionPool(host='api.openai.com', port=443): Read timed out. (read timeout=600)",
      "\nThe above exception was the direct cause of the following exception:\n",
      "\u001b[0;31mAPIConnectionError\u001b[0m                        Traceback (most recent call last)",
      "Input \u001b[0;32mIn [62]\u001b[0m, in \u001b[0;36m<cell line: 6>\u001b[0;34m()\u001b[0m\n\u001b[1;32m     19\u001b[0m prompt_q \u001b[38;5;241m+\u001b[39m\u001b[38;5;241m=\u001b[39m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mLet\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124ms think step by step\u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m\n\u001b[1;32m     21\u001b[0m prompt_full \u001b[38;5;241m=\u001b[39m \u001b[38;5;124m'\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;241m.\u001b[39mjoin(prompt_complex) \u001b[38;5;241m+\u001b[39m prompt_q\n\u001b[0;32m---> 22\u001b[0m response \u001b[38;5;241m=\u001b[39m \u001b[43mopenai\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mCompletion\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mcreate\u001b[49m\u001b[43m(\u001b[49m\u001b[43mmodel\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mtext-davinci-002\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\n\u001b[1;32m     23\u001b[0m \u001b[43m                                    \u001b[49m\u001b[43mprompt\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mprompt_full\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\n\u001b[1;32m     24\u001b[0m \u001b[43m                                    \u001b[49m\u001b[43mtemperature\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;241;43m0.5\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\n\u001b[1;32m     25\u001b[0m \u001b[43m                                    \u001b[49m\u001b[43mmax_tokens\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;241;43m256\u001b[39;49m\u001b[43m,\u001b[49m\n\u001b[1;32m     26\u001b[0m \u001b[43m                                    \u001b[49m\u001b[43mn\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;241;43m50\u001b[39;49m\n\u001b[1;32m     27\u001b[0m \u001b[43m                                   \u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m     29\u001b[0m ans_strs \u001b[38;5;241m=\u001b[39m []\n\u001b[1;32m     30\u001b[0m \u001b[38;5;28;01mfor\u001b[39;00m ans \u001b[38;5;129;01min\u001b[39;00m response[\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mchoices\u001b[39m\u001b[38;5;124m'\u001b[39m]: ans_strs\u001b[38;5;241m.\u001b[39mappend(ans[\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mtext\u001b[39m\u001b[38;5;124m'\u001b[39m])\n",
      "File \u001b[0;32m~/miniconda3/envs/dl_py3.8/lib/python3.8/site-packages/openai/api_resources/completion.py:25\u001b[0m, in \u001b[0;36mCompletion.create\u001b[0;34m(cls, *args, **kwargs)\u001b[0m\n\u001b[1;32m     23\u001b[0m \u001b[38;5;28;01mwhile\u001b[39;00m \u001b[38;5;28;01mTrue\u001b[39;00m:\n\u001b[1;32m     24\u001b[0m     \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[0;32m---> 25\u001b[0m         \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43msuper\u001b[39;49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mcreate\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m     26\u001b[0m     \u001b[38;5;28;01mexcept\u001b[39;00m TryAgain \u001b[38;5;28;01mas\u001b[39;00m e:\n\u001b[1;32m     27\u001b[0m         \u001b[38;5;28;01mif\u001b[39;00m timeout \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m \u001b[38;5;129;01mand\u001b[39;00m time\u001b[38;5;241m.\u001b[39mtime() \u001b[38;5;241m>\u001b[39m start \u001b[38;5;241m+\u001b[39m timeout:\n",
      "File \u001b[0;32m~/miniconda3/envs/dl_py3.8/lib/python3.8/site-packages/openai/api_resources/abstract/engine_api_resource.py:115\u001b[0m, in \u001b[0;36mEngineAPIResource.create\u001b[0;34m(cls, api_key, api_base, api_type, request_id, api_version, organization, **params)\u001b[0m\n\u001b[1;32m    107\u001b[0m requestor \u001b[38;5;241m=\u001b[39m api_requestor\u001b[38;5;241m.\u001b[39mAPIRequestor(\n\u001b[1;32m    108\u001b[0m     api_key,\n\u001b[1;32m    109\u001b[0m     api_base\u001b[38;5;241m=\u001b[39mapi_base,\n\u001b[0;32m   (...)\u001b[0m\n\u001b[1;32m    112\u001b[0m     organization\u001b[38;5;241m=\u001b[39morganization,\n\u001b[1;32m    113\u001b[0m )\n\u001b[1;32m    114\u001b[0m url \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mcls\u001b[39m\u001b[38;5;241m.\u001b[39mclass_url(engine, api_type, api_version)\n\u001b[0;32m--> 115\u001b[0m response, _, api_key \u001b[38;5;241m=\u001b[39m \u001b[43mrequestor\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mrequest\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m    116\u001b[0m \u001b[43m    \u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mpost\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m,\u001b[49m\n\u001b[1;32m    117\u001b[0m \u001b[43m    \u001b[49m\u001b[43murl\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    118\u001b[0m \u001b[43m    \u001b[49m\u001b[43mparams\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mparams\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    119\u001b[0m \u001b[43m    \u001b[49m\u001b[43mheaders\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mheaders\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    120\u001b[0m \u001b[43m    \u001b[49m\u001b[43mstream\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mstream\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    121\u001b[0m \u001b[43m    \u001b[49m\u001b[43mrequest_id\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mrequest_id\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    122\u001b[0m \u001b[43m\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    124\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m stream:\n\u001b[1;32m    125\u001b[0m     \u001b[38;5;66;03m# must be an iterator\u001b[39;00m\n\u001b[1;32m    126\u001b[0m     \u001b[38;5;28;01massert\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;28misinstance\u001b[39m(response, OpenAIResponse)\n",
      "File \u001b[0;32m~/miniconda3/envs/dl_py3.8/lib/python3.8/site-packages/openai/api_requestor.py:112\u001b[0m, in \u001b[0;36mAPIRequestor.request\u001b[0;34m(self, method, url, params, headers, files, stream, request_id)\u001b[0m\n\u001b[1;32m    102\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mrequest\u001b[39m(\n\u001b[1;32m    103\u001b[0m     \u001b[38;5;28mself\u001b[39m,\n\u001b[1;32m    104\u001b[0m     method,\n\u001b[0;32m   (...)\u001b[0m\n\u001b[1;32m    110\u001b[0m     request_id: Optional[\u001b[38;5;28mstr\u001b[39m] \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mNone\u001b[39;00m,\n\u001b[1;32m    111\u001b[0m ) \u001b[38;5;241m-\u001b[39m\u001b[38;5;241m>\u001b[39m Tuple[Union[OpenAIResponse, Iterator[OpenAIResponse]], \u001b[38;5;28mbool\u001b[39m, \u001b[38;5;28mstr\u001b[39m]:\n\u001b[0;32m--> 112\u001b[0m     result \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mrequest_raw\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m    113\u001b[0m \u001b[43m        \u001b[49m\u001b[43mmethod\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mlower\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    114\u001b[0m \u001b[43m        \u001b[49m\u001b[43murl\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    115\u001b[0m \u001b[43m        \u001b[49m\u001b[43mparams\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mparams\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    116\u001b[0m \u001b[43m        \u001b[49m\u001b[43msupplied_headers\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mheaders\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    117\u001b[0m \u001b[43m        \u001b[49m\u001b[43mfiles\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mfiles\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    118\u001b[0m \u001b[43m        \u001b[49m\u001b[43mstream\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mstream\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    119\u001b[0m \u001b[43m        \u001b[49m\u001b[43mrequest_id\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mrequest_id\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    120\u001b[0m \u001b[43m    \u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    121\u001b[0m     resp, got_stream \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_interpret_response(result, stream)\n\u001b[1;32m    122\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m resp, got_stream, \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mapi_key\n",
      "File \u001b[0;32m~/miniconda3/envs/dl_py3.8/lib/python3.8/site-packages/openai/api_requestor.py:301\u001b[0m, in \u001b[0;36mAPIRequestor.request_raw\u001b[0;34m(self, method, url, params, supplied_headers, files, stream, request_id)\u001b[0m\n\u001b[1;32m    291\u001b[0m     result \u001b[38;5;241m=\u001b[39m _thread_context\u001b[38;5;241m.\u001b[39msession\u001b[38;5;241m.\u001b[39mrequest(\n\u001b[1;32m    292\u001b[0m         method,\n\u001b[1;32m    293\u001b[0m         abs_url,\n\u001b[0;32m   (...)\u001b[0m\n\u001b[1;32m    298\u001b[0m         timeout\u001b[38;5;241m=\u001b[39mTIMEOUT_SECS,\n\u001b[1;32m    299\u001b[0m     )\n\u001b[1;32m    300\u001b[0m \u001b[38;5;28;01mexcept\u001b[39;00m requests\u001b[38;5;241m.\u001b[39mexceptions\u001b[38;5;241m.\u001b[39mRequestException \u001b[38;5;28;01mas\u001b[39;00m e:\n\u001b[0;32m--> 301\u001b[0m     \u001b[38;5;28;01mraise\u001b[39;00m error\u001b[38;5;241m.\u001b[39mAPIConnectionError(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mError communicating with OpenAI\u001b[39m\u001b[38;5;124m\"\u001b[39m) \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01me\u001b[39;00m\n\u001b[1;32m    302\u001b[0m util\u001b[38;5;241m.\u001b[39mlog_info(\n\u001b[1;32m    303\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mOpenAI API response\u001b[39m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m    304\u001b[0m     path\u001b[38;5;241m=\u001b[39mabs_url,\n\u001b[1;32m    305\u001b[0m     response_code\u001b[38;5;241m=\u001b[39mresult\u001b[38;5;241m.\u001b[39mstatus_code,\n\u001b[1;32m    306\u001b[0m     processing_ms\u001b[38;5;241m=\u001b[39mresult\u001b[38;5;241m.\u001b[39mheaders\u001b[38;5;241m.\u001b[39mget(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mOpenAI-Processing-Ms\u001b[39m\u001b[38;5;124m\"\u001b[39m),\n\u001b[1;32m    307\u001b[0m )\n\u001b[1;32m    308\u001b[0m \u001b[38;5;66;03m# Don't read the whole stream for debug logging unless necessary.\u001b[39;00m\n",
      "\u001b[0;31mAPIConnectionError\u001b[0m: Error communicating with OpenAI"
     ]
    }
   ],
   "source": [
    "q_total = []\n",
    "ans_total = []\n",
    "ans_gold_total = []\n",
    "i = 0\n",
    "    \n",
    "for case in tqdm(dev_data_200):\n",
    "    q = case['Problem']\n",
    "    c = case['options']\n",
    "    a = case['Rationale']\n",
    "    cr = case['correct']\n",
    "\n",
    "    ans_dict = get_ans_dict(c)\n",
    "\n",
    "    a += ' ' + ans_dict[cr]\n",
    "\n",
    "    prompt_q = '' \n",
    "    prompt_q += '\\nQuestion: ' + q + '\\n'\n",
    "    prompt_q += 'Options: ' + c + '\\n'\n",
    "    prompt_q += \"Let's think step by step\\n\"\n",
    "\n",
    "    prompt_full = ''.join(prompt_complex) + prompt_q\n",
    "    response = openai.Completion.create(model=\"text-davinci-002\", \n",
    "                                        prompt=prompt_full, \n",
    "                                        temperature=0.5, \n",
    "                                        max_tokens=256,\n",
    "                                        n=50\n",
    "                                       )\n",
    "        \n",
    "    ans_strs = []\n",
    "    for ans in response['choices']: ans_strs.append(ans['text'])\n",
    "    ans_total.append(ans_strs)\n",
    "    q_total.append(q)\n",
    "    ans_gold_total.append(a)\n",
    "    i += 1\n",
    "    time.sleep(10)\n",
    "    # if(i == 2): break"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 76,
   "id": "61c4c63a-5616-4655-976f-9d0869766f0f",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Correct 49 total 98 acc 0.5000\n"
     ]
    }
   ],
   "source": [
    "total, acc = 0, 0\n",
    "\n",
    "with open('outputs/dev_consistency_complex.txt', 'w') as fd:\n",
    "    for q, a, pred in zip(q_total, ans_gold_total, ans_total):\n",
    "        fd.write('Q: ' + q + ' [EOQ]\\n--\\n')\n",
    "        fd.write('A_model:\\n')\n",
    "        for p in pred:\n",
    "            fd.write(p + ' [EOA]\\n\\n')\n",
    "        majority = find_majority(pred)\n",
    "        fd.write('\\nMajority = ' + majority + '\\n')\n",
    "        fd.write('--\\n')\n",
    "        fd.write('A_gold:\\n' + a + '\\n')\n",
    "        fd.write('\\n\\n----\\n\\n')\n",
    "        \n",
    "        if(test_answer(majority, a)):\n",
    "            acc += 1\n",
    "        total += 1\n",
    "\n",
    "print('Correct %d total %d acc %.4f' % (acc, total, float(acc) / total))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 84,
   "id": "77aaf99a-0dcb-4ec9-a618-dd5b307045b8",
   "metadata": {
    "collapsed": true,
    "jupyter": {
     "outputs_hidden": true
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▎| 199/200 [45:25<00:13, 13.69s/it]\n"
     ]
    },
    {
     "ename": "APIConnectionError",
     "evalue": "Error communicating with OpenAI",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mtimeout\u001b[0m                                   Traceback (most recent call last)",
      "File \u001b[0;32m~/miniconda3/envs/dl_py3.8/lib/python3.8/site-packages/urllib3/connectionpool.py:449\u001b[0m, in \u001b[0;36mHTTPConnectionPool._make_request\u001b[0;34m(self, conn, method, url, timeout, chunked, **httplib_request_kw)\u001b[0m\n\u001b[1;32m    445\u001b[0m         \u001b[38;5;28;01mexcept\u001b[39;00m \u001b[38;5;167;01mBaseException\u001b[39;00m \u001b[38;5;28;01mas\u001b[39;00m e:\n\u001b[1;32m    446\u001b[0m             \u001b[38;5;66;03m# Remove the TypeError from the exception chain in\u001b[39;00m\n\u001b[1;32m    447\u001b[0m             \u001b[38;5;66;03m# Python 3 (including for exceptions like SystemExit).\u001b[39;00m\n\u001b[1;32m    448\u001b[0m             \u001b[38;5;66;03m# Otherwise it looks like a bug in the code.\u001b[39;00m\n\u001b[0;32m--> 449\u001b[0m             \u001b[43msix\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mraise_from\u001b[49m\u001b[43m(\u001b[49m\u001b[43me\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;28;43;01mNone\u001b[39;49;00m\u001b[43m)\u001b[49m\n\u001b[1;32m    450\u001b[0m \u001b[38;5;28;01mexcept\u001b[39;00m (SocketTimeout, BaseSSLError, SocketError) \u001b[38;5;28;01mas\u001b[39;00m e:\n",
      "File \u001b[0;32m<string>:3\u001b[0m, in \u001b[0;36mraise_from\u001b[0;34m(value, from_value)\u001b[0m\n",
      "File \u001b[0;32m~/miniconda3/envs/dl_py3.8/lib/python3.8/site-packages/urllib3/connectionpool.py:444\u001b[0m, in \u001b[0;36mHTTPConnectionPool._make_request\u001b[0;34m(self, conn, method, url, timeout, chunked, **httplib_request_kw)\u001b[0m\n\u001b[1;32m    443\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[0;32m--> 444\u001b[0m     httplib_response \u001b[38;5;241m=\u001b[39m \u001b[43mconn\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mgetresponse\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    445\u001b[0m \u001b[38;5;28;01mexcept\u001b[39;00m \u001b[38;5;167;01mBaseException\u001b[39;00m \u001b[38;5;28;01mas\u001b[39;00m e:\n\u001b[1;32m    446\u001b[0m     \u001b[38;5;66;03m# Remove the TypeError from the exception chain in\u001b[39;00m\n\u001b[1;32m    447\u001b[0m     \u001b[38;5;66;03m# Python 3 (including for exceptions like SystemExit).\u001b[39;00m\n\u001b[1;32m    448\u001b[0m     \u001b[38;5;66;03m# Otherwise it looks like a bug in the code.\u001b[39;00m\n",
      "File \u001b[0;32m~/miniconda3/envs/dl_py3.8/lib/python3.8/http/client.py:1348\u001b[0m, in \u001b[0;36mHTTPConnection.getresponse\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m   1347\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[0;32m-> 1348\u001b[0m     \u001b[43mresponse\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mbegin\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m   1349\u001b[0m \u001b[38;5;28;01mexcept\u001b[39;00m \u001b[38;5;167;01mConnectionError\u001b[39;00m:\n",
      "File \u001b[0;32m~/miniconda3/envs/dl_py3.8/lib/python3.8/http/client.py:316\u001b[0m, in \u001b[0;36mHTTPResponse.begin\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m    315\u001b[0m \u001b[38;5;28;01mwhile\u001b[39;00m \u001b[38;5;28;01mTrue\u001b[39;00m:\n\u001b[0;32m--> 316\u001b[0m     version, status, reason \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_read_status\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    317\u001b[0m     \u001b[38;5;28;01mif\u001b[39;00m status \u001b[38;5;241m!=\u001b[39m CONTINUE:\n",
      "File \u001b[0;32m~/miniconda3/envs/dl_py3.8/lib/python3.8/http/client.py:277\u001b[0m, in \u001b[0;36mHTTPResponse._read_status\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m    276\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21m_read_status\u001b[39m(\u001b[38;5;28mself\u001b[39m):\n\u001b[0;32m--> 277\u001b[0m     line \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mstr\u001b[39m(\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mfp\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mreadline\u001b[49m\u001b[43m(\u001b[49m\u001b[43m_MAXLINE\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m+\u001b[39;49m\u001b[43m \u001b[49m\u001b[38;5;241;43m1\u001b[39;49m\u001b[43m)\u001b[49m, \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124miso-8859-1\u001b[39m\u001b[38;5;124m\"\u001b[39m)\n\u001b[1;32m    278\u001b[0m     \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mlen\u001b[39m(line) \u001b[38;5;241m>\u001b[39m _MAXLINE:\n",
      "File \u001b[0;32m~/miniconda3/envs/dl_py3.8/lib/python3.8/socket.py:669\u001b[0m, in \u001b[0;36mSocketIO.readinto\u001b[0;34m(self, b)\u001b[0m\n\u001b[1;32m    668\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[0;32m--> 669\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_sock\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mrecv_into\u001b[49m\u001b[43m(\u001b[49m\u001b[43mb\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    670\u001b[0m \u001b[38;5;28;01mexcept\u001b[39;00m timeout:\n",
      "File \u001b[0;32m~/miniconda3/envs/dl_py3.8/lib/python3.8/ssl.py:1241\u001b[0m, in \u001b[0;36mSSLSocket.recv_into\u001b[0;34m(self, buffer, nbytes, flags)\u001b[0m\n\u001b[1;32m   1238\u001b[0m         \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;167;01mValueError\u001b[39;00m(\n\u001b[1;32m   1239\u001b[0m           \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mnon-zero flags not allowed in calls to recv_into() on \u001b[39m\u001b[38;5;132;01m%s\u001b[39;00m\u001b[38;5;124m\"\u001b[39m \u001b[38;5;241m%\u001b[39m\n\u001b[1;32m   1240\u001b[0m           \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m\u001b[38;5;18m__class__\u001b[39m)\n\u001b[0;32m-> 1241\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mread\u001b[49m\u001b[43m(\u001b[49m\u001b[43mnbytes\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mbuffer\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m   1242\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n",
      "File \u001b[0;32m~/miniconda3/envs/dl_py3.8/lib/python3.8/ssl.py:1099\u001b[0m, in \u001b[0;36mSSLSocket.read\u001b[0;34m(self, len, buffer)\u001b[0m\n\u001b[1;32m   1098\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m buffer \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m:\n\u001b[0;32m-> 1099\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_sslobj\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mread\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;28;43mlen\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mbuffer\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m   1100\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n",
      "\u001b[0;31mtimeout\u001b[0m: The read operation timed out",
      "\nDuring handling of the above exception, another exception occurred:\n",
      "\u001b[0;31mReadTimeoutError\u001b[0m                          Traceback (most recent call last)",
      "File \u001b[0;32m~/miniconda3/envs/dl_py3.8/lib/python3.8/site-packages/requests/adapters.py:489\u001b[0m, in \u001b[0;36mHTTPAdapter.send\u001b[0;34m(self, request, stream, timeout, verify, cert, proxies)\u001b[0m\n\u001b[1;32m    488\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m chunked:\n\u001b[0;32m--> 489\u001b[0m     resp \u001b[38;5;241m=\u001b[39m \u001b[43mconn\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43murlopen\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m    490\u001b[0m \u001b[43m        \u001b[49m\u001b[43mmethod\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mrequest\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mmethod\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    491\u001b[0m \u001b[43m        \u001b[49m\u001b[43murl\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43murl\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    492\u001b[0m \u001b[43m        \u001b[49m\u001b[43mbody\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mrequest\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mbody\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    493\u001b[0m \u001b[43m        \u001b[49m\u001b[43mheaders\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mrequest\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mheaders\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    494\u001b[0m \u001b[43m        \u001b[49m\u001b[43mredirect\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;28;43;01mFalse\u001b[39;49;00m\u001b[43m,\u001b[49m\n\u001b[1;32m    495\u001b[0m \u001b[43m        \u001b[49m\u001b[43massert_same_host\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;28;43;01mFalse\u001b[39;49;00m\u001b[43m,\u001b[49m\n\u001b[1;32m    496\u001b[0m \u001b[43m        \u001b[49m\u001b[43mpreload_content\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;28;43;01mFalse\u001b[39;49;00m\u001b[43m,\u001b[49m\n\u001b[1;32m    497\u001b[0m \u001b[43m        \u001b[49m\u001b[43mdecode_content\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;28;43;01mFalse\u001b[39;49;00m\u001b[43m,\u001b[49m\n\u001b[1;32m    498\u001b[0m \u001b[43m        \u001b[49m\u001b[43mretries\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mmax_retries\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    499\u001b[0m \u001b[43m        \u001b[49m\u001b[43mtimeout\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mtimeout\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    500\u001b[0m \u001b[43m    \u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    502\u001b[0m \u001b[38;5;66;03m# Send the request.\u001b[39;00m\n\u001b[1;32m    503\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n",
      "File \u001b[0;32m~/miniconda3/envs/dl_py3.8/lib/python3.8/site-packages/urllib3/connectionpool.py:787\u001b[0m, in \u001b[0;36mHTTPConnectionPool.urlopen\u001b[0;34m(self, method, url, body, headers, retries, redirect, assert_same_host, timeout, pool_timeout, release_conn, chunked, body_pos, **response_kw)\u001b[0m\n\u001b[1;32m    785\u001b[0m     e \u001b[38;5;241m=\u001b[39m ProtocolError(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mConnection aborted.\u001b[39m\u001b[38;5;124m\"\u001b[39m, e)\n\u001b[0;32m--> 787\u001b[0m retries \u001b[38;5;241m=\u001b[39m \u001b[43mretries\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mincrement\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m    788\u001b[0m \u001b[43m    \u001b[49m\u001b[43mmethod\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43murl\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43merror\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43me\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43m_pool\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43m_stacktrace\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43msys\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mexc_info\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\u001b[43m[\u001b[49m\u001b[38;5;241;43m2\u001b[39;49m\u001b[43m]\u001b[49m\n\u001b[1;32m    789\u001b[0m \u001b[43m\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    790\u001b[0m retries\u001b[38;5;241m.\u001b[39msleep()\n",
      "File \u001b[0;32m~/miniconda3/envs/dl_py3.8/lib/python3.8/site-packages/urllib3/util/retry.py:550\u001b[0m, in \u001b[0;36mRetry.increment\u001b[0;34m(self, method, url, response, error, _pool, _stacktrace)\u001b[0m\n\u001b[1;32m    549\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m read \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;28;01mFalse\u001b[39;00m \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_is_method_retryable(method):\n\u001b[0;32m--> 550\u001b[0m     \u001b[38;5;28;01mraise\u001b[39;00m \u001b[43msix\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mreraise\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;28;43mtype\u001b[39;49m\u001b[43m(\u001b[49m\u001b[43merror\u001b[49m\u001b[43m)\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43merror\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43m_stacktrace\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    551\u001b[0m \u001b[38;5;28;01melif\u001b[39;00m read \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m:\n",
      "File \u001b[0;32m~/miniconda3/envs/dl_py3.8/lib/python3.8/site-packages/urllib3/packages/six.py:770\u001b[0m, in \u001b[0;36mreraise\u001b[0;34m(tp, value, tb)\u001b[0m\n\u001b[1;32m    769\u001b[0m         \u001b[38;5;28;01mraise\u001b[39;00m value\u001b[38;5;241m.\u001b[39mwith_traceback(tb)\n\u001b[0;32m--> 770\u001b[0m     \u001b[38;5;28;01mraise\u001b[39;00m value\n\u001b[1;32m    771\u001b[0m \u001b[38;5;28;01mfinally\u001b[39;00m:\n",
      "File \u001b[0;32m~/miniconda3/envs/dl_py3.8/lib/python3.8/site-packages/urllib3/connectionpool.py:703\u001b[0m, in \u001b[0;36mHTTPConnectionPool.urlopen\u001b[0;34m(self, method, url, body, headers, retries, redirect, assert_same_host, timeout, pool_timeout, release_conn, chunked, body_pos, **response_kw)\u001b[0m\n\u001b[1;32m    702\u001b[0m \u001b[38;5;66;03m# Make the request on the httplib connection object.\u001b[39;00m\n\u001b[0;32m--> 703\u001b[0m httplib_response \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_make_request\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m    704\u001b[0m \u001b[43m    \u001b[49m\u001b[43mconn\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    705\u001b[0m \u001b[43m    \u001b[49m\u001b[43mmethod\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    706\u001b[0m \u001b[43m    \u001b[49m\u001b[43murl\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    707\u001b[0m \u001b[43m    \u001b[49m\u001b[43mtimeout\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mtimeout_obj\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    708\u001b[0m \u001b[43m    \u001b[49m\u001b[43mbody\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mbody\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    709\u001b[0m \u001b[43m    \u001b[49m\u001b[43mheaders\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mheaders\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    710\u001b[0m \u001b[43m    \u001b[49m\u001b[43mchunked\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mchunked\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    711\u001b[0m \u001b[43m\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    713\u001b[0m \u001b[38;5;66;03m# If we're going to release the connection in ``finally:``, then\u001b[39;00m\n\u001b[1;32m    714\u001b[0m \u001b[38;5;66;03m# the response doesn't need to know about the connection. Otherwise\u001b[39;00m\n\u001b[1;32m    715\u001b[0m \u001b[38;5;66;03m# it will also try to release it and we'll have a double-release\u001b[39;00m\n\u001b[1;32m    716\u001b[0m \u001b[38;5;66;03m# mess.\u001b[39;00m\n",
      "File \u001b[0;32m~/miniconda3/envs/dl_py3.8/lib/python3.8/site-packages/urllib3/connectionpool.py:451\u001b[0m, in \u001b[0;36mHTTPConnectionPool._make_request\u001b[0;34m(self, conn, method, url, timeout, chunked, **httplib_request_kw)\u001b[0m\n\u001b[1;32m    450\u001b[0m \u001b[38;5;28;01mexcept\u001b[39;00m (SocketTimeout, BaseSSLError, SocketError) \u001b[38;5;28;01mas\u001b[39;00m e:\n\u001b[0;32m--> 451\u001b[0m     \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_raise_timeout\u001b[49m\u001b[43m(\u001b[49m\u001b[43merr\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43me\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43murl\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43murl\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mtimeout_value\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mread_timeout\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    452\u001b[0m     \u001b[38;5;28;01mraise\u001b[39;00m\n",
      "File \u001b[0;32m~/miniconda3/envs/dl_py3.8/lib/python3.8/site-packages/urllib3/connectionpool.py:340\u001b[0m, in \u001b[0;36mHTTPConnectionPool._raise_timeout\u001b[0;34m(self, err, url, timeout_value)\u001b[0m\n\u001b[1;32m    339\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28misinstance\u001b[39m(err, SocketTimeout):\n\u001b[0;32m--> 340\u001b[0m     \u001b[38;5;28;01mraise\u001b[39;00m ReadTimeoutError(\n\u001b[1;32m    341\u001b[0m         \u001b[38;5;28mself\u001b[39m, url, \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mRead timed out. (read timeout=\u001b[39m\u001b[38;5;132;01m%s\u001b[39;00m\u001b[38;5;124m)\u001b[39m\u001b[38;5;124m\"\u001b[39m \u001b[38;5;241m%\u001b[39m timeout_value\n\u001b[1;32m    342\u001b[0m     )\n\u001b[1;32m    344\u001b[0m \u001b[38;5;66;03m# See the above comment about EAGAIN in Python 3. In Python 2 we have\u001b[39;00m\n\u001b[1;32m    345\u001b[0m \u001b[38;5;66;03m# to specifically catch it and throw the timeout error\u001b[39;00m\n",
      "\u001b[0;31mReadTimeoutError\u001b[0m: HTTPSConnectionPool(host='api.openai.com', port=443): Read timed out. (read timeout=600)",
      "\nDuring handling of the above exception, another exception occurred:\n",
      "\u001b[0;31mReadTimeout\u001b[0m                               Traceback (most recent call last)",
      "File \u001b[0;32m~/miniconda3/envs/dl_py3.8/lib/python3.8/site-packages/openai/api_requestor.py:291\u001b[0m, in \u001b[0;36mAPIRequestor.request_raw\u001b[0;34m(self, method, url, params, supplied_headers, files, stream, request_id)\u001b[0m\n\u001b[1;32m    290\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[0;32m--> 291\u001b[0m     result \u001b[38;5;241m=\u001b[39m \u001b[43m_thread_context\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43msession\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mrequest\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m    292\u001b[0m \u001b[43m        \u001b[49m\u001b[43mmethod\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    293\u001b[0m \u001b[43m        \u001b[49m\u001b[43mabs_url\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    294\u001b[0m \u001b[43m        \u001b[49m\u001b[43mheaders\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mheaders\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    295\u001b[0m \u001b[43m        \u001b[49m\u001b[43mdata\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mdata\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    296\u001b[0m \u001b[43m        \u001b[49m\u001b[43mfiles\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mfiles\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    297\u001b[0m \u001b[43m        \u001b[49m\u001b[43mstream\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mstream\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    298\u001b[0m \u001b[43m        \u001b[49m\u001b[43mtimeout\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mTIMEOUT_SECS\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    299\u001b[0m \u001b[43m    \u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    300\u001b[0m \u001b[38;5;28;01mexcept\u001b[39;00m requests\u001b[38;5;241m.\u001b[39mexceptions\u001b[38;5;241m.\u001b[39mRequestException \u001b[38;5;28;01mas\u001b[39;00m e:\n",
      "File \u001b[0;32m~/miniconda3/envs/dl_py3.8/lib/python3.8/site-packages/requests/sessions.py:587\u001b[0m, in \u001b[0;36mSession.request\u001b[0;34m(self, method, url, params, data, headers, cookies, files, auth, timeout, allow_redirects, proxies, hooks, stream, verify, cert, json)\u001b[0m\n\u001b[1;32m    586\u001b[0m send_kwargs\u001b[38;5;241m.\u001b[39mupdate(settings)\n\u001b[0;32m--> 587\u001b[0m resp \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43msend\u001b[49m\u001b[43m(\u001b[49m\u001b[43mprep\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43msend_kwargs\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    589\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m resp\n",
      "File \u001b[0;32m~/miniconda3/envs/dl_py3.8/lib/python3.8/site-packages/requests/sessions.py:701\u001b[0m, in \u001b[0;36mSession.send\u001b[0;34m(self, request, **kwargs)\u001b[0m\n\u001b[1;32m    700\u001b[0m \u001b[38;5;66;03m# Send the request\u001b[39;00m\n\u001b[0;32m--> 701\u001b[0m r \u001b[38;5;241m=\u001b[39m \u001b[43madapter\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43msend\u001b[49m\u001b[43m(\u001b[49m\u001b[43mrequest\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    703\u001b[0m \u001b[38;5;66;03m# Total elapsed time of the request (approximately)\u001b[39;00m\n",
      "File \u001b[0;32m~/miniconda3/envs/dl_py3.8/lib/python3.8/site-packages/requests/adapters.py:578\u001b[0m, in \u001b[0;36mHTTPAdapter.send\u001b[0;34m(self, request, stream, timeout, verify, cert, proxies)\u001b[0m\n\u001b[1;32m    577\u001b[0m \u001b[38;5;28;01melif\u001b[39;00m \u001b[38;5;28misinstance\u001b[39m(e, ReadTimeoutError):\n\u001b[0;32m--> 578\u001b[0m     \u001b[38;5;28;01mraise\u001b[39;00m ReadTimeout(e, request\u001b[38;5;241m=\u001b[39mrequest)\n\u001b[1;32m    579\u001b[0m \u001b[38;5;28;01melif\u001b[39;00m \u001b[38;5;28misinstance\u001b[39m(e, _InvalidHeader):\n",
      "\u001b[0;31mReadTimeout\u001b[0m: HTTPSConnectionPool(host='api.openai.com', port=443): Read timed out. (read timeout=600)",
      "\nThe above exception was the direct cause of the following exception:\n",
      "\u001b[0;31mAPIConnectionError\u001b[0m                        Traceback (most recent call last)",
      "Input \u001b[0;32mIn [84]\u001b[0m, in \u001b[0;36m<cell line: 6>\u001b[0;34m()\u001b[0m\n\u001b[1;32m     22\u001b[0m prompt_q \u001b[38;5;241m+\u001b[39m\u001b[38;5;241m=\u001b[39m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mLet\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124ms think step by step\u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m\n\u001b[1;32m     24\u001b[0m prompt_full \u001b[38;5;241m=\u001b[39m \u001b[38;5;124m'\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;241m.\u001b[39mjoin(prompt_complex) \u001b[38;5;241m+\u001b[39m prompt_q\n\u001b[0;32m---> 25\u001b[0m response \u001b[38;5;241m=\u001b[39m \u001b[43mopenai\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mCompletion\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mcreate\u001b[49m\u001b[43m(\u001b[49m\u001b[43mmodel\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mtext-davinci-002\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\n\u001b[1;32m     26\u001b[0m \u001b[43m                                    \u001b[49m\u001b[43mprompt\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mprompt_full\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\n\u001b[1;32m     27\u001b[0m \u001b[43m                                    \u001b[49m\u001b[43mtemperature\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;241;43m0.5\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\n\u001b[1;32m     28\u001b[0m \u001b[43m                                    \u001b[49m\u001b[43mmax_tokens\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;241;43m256\u001b[39;49m\u001b[43m,\u001b[49m\n\u001b[1;32m     29\u001b[0m \u001b[43m                                    \u001b[49m\u001b[43mn\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;241;43m50\u001b[39;49m\n\u001b[1;32m     30\u001b[0m \u001b[43m                                   \u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m     32\u001b[0m ans_strs \u001b[38;5;241m=\u001b[39m []\n\u001b[1;32m     33\u001b[0m \u001b[38;5;28;01mfor\u001b[39;00m ans \u001b[38;5;129;01min\u001b[39;00m response[\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mchoices\u001b[39m\u001b[38;5;124m'\u001b[39m]: ans_strs\u001b[38;5;241m.\u001b[39mappend(ans[\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mtext\u001b[39m\u001b[38;5;124m'\u001b[39m])\n",
      "File \u001b[0;32m~/miniconda3/envs/dl_py3.8/lib/python3.8/site-packages/openai/api_resources/completion.py:25\u001b[0m, in \u001b[0;36mCompletion.create\u001b[0;34m(cls, *args, **kwargs)\u001b[0m\n\u001b[1;32m     23\u001b[0m \u001b[38;5;28;01mwhile\u001b[39;00m \u001b[38;5;28;01mTrue\u001b[39;00m:\n\u001b[1;32m     24\u001b[0m     \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[0;32m---> 25\u001b[0m         \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43msuper\u001b[39;49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mcreate\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m     26\u001b[0m     \u001b[38;5;28;01mexcept\u001b[39;00m TryAgain \u001b[38;5;28;01mas\u001b[39;00m e:\n\u001b[1;32m     27\u001b[0m         \u001b[38;5;28;01mif\u001b[39;00m timeout \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m \u001b[38;5;129;01mand\u001b[39;00m time\u001b[38;5;241m.\u001b[39mtime() \u001b[38;5;241m>\u001b[39m start \u001b[38;5;241m+\u001b[39m timeout:\n",
      "File \u001b[0;32m~/miniconda3/envs/dl_py3.8/lib/python3.8/site-packages/openai/api_resources/abstract/engine_api_resource.py:115\u001b[0m, in \u001b[0;36mEngineAPIResource.create\u001b[0;34m(cls, api_key, api_base, api_type, request_id, api_version, organization, **params)\u001b[0m\n\u001b[1;32m    107\u001b[0m requestor \u001b[38;5;241m=\u001b[39m api_requestor\u001b[38;5;241m.\u001b[39mAPIRequestor(\n\u001b[1;32m    108\u001b[0m     api_key,\n\u001b[1;32m    109\u001b[0m     api_base\u001b[38;5;241m=\u001b[39mapi_base,\n\u001b[0;32m   (...)\u001b[0m\n\u001b[1;32m    112\u001b[0m     organization\u001b[38;5;241m=\u001b[39morganization,\n\u001b[1;32m    113\u001b[0m )\n\u001b[1;32m    114\u001b[0m url \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mcls\u001b[39m\u001b[38;5;241m.\u001b[39mclass_url(engine, api_type, api_version)\n\u001b[0;32m--> 115\u001b[0m response, _, api_key \u001b[38;5;241m=\u001b[39m \u001b[43mrequestor\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mrequest\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m    116\u001b[0m \u001b[43m    \u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mpost\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m,\u001b[49m\n\u001b[1;32m    117\u001b[0m \u001b[43m    \u001b[49m\u001b[43murl\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    118\u001b[0m \u001b[43m    \u001b[49m\u001b[43mparams\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mparams\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    119\u001b[0m \u001b[43m    \u001b[49m\u001b[43mheaders\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mheaders\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    120\u001b[0m \u001b[43m    \u001b[49m\u001b[43mstream\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mstream\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    121\u001b[0m \u001b[43m    \u001b[49m\u001b[43mrequest_id\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mrequest_id\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    122\u001b[0m \u001b[43m\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    124\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m stream:\n\u001b[1;32m    125\u001b[0m     \u001b[38;5;66;03m# must be an iterator\u001b[39;00m\n\u001b[1;32m    126\u001b[0m     \u001b[38;5;28;01massert\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;28misinstance\u001b[39m(response, OpenAIResponse)\n",
      "File \u001b[0;32m~/miniconda3/envs/dl_py3.8/lib/python3.8/site-packages/openai/api_requestor.py:112\u001b[0m, in \u001b[0;36mAPIRequestor.request\u001b[0;34m(self, method, url, params, headers, files, stream, request_id)\u001b[0m\n\u001b[1;32m    102\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mrequest\u001b[39m(\n\u001b[1;32m    103\u001b[0m     \u001b[38;5;28mself\u001b[39m,\n\u001b[1;32m    104\u001b[0m     method,\n\u001b[0;32m   (...)\u001b[0m\n\u001b[1;32m    110\u001b[0m     request_id: Optional[\u001b[38;5;28mstr\u001b[39m] \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mNone\u001b[39;00m,\n\u001b[1;32m    111\u001b[0m ) \u001b[38;5;241m-\u001b[39m\u001b[38;5;241m>\u001b[39m Tuple[Union[OpenAIResponse, Iterator[OpenAIResponse]], \u001b[38;5;28mbool\u001b[39m, \u001b[38;5;28mstr\u001b[39m]:\n\u001b[0;32m--> 112\u001b[0m     result \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mrequest_raw\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m    113\u001b[0m \u001b[43m        \u001b[49m\u001b[43mmethod\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mlower\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    114\u001b[0m \u001b[43m        \u001b[49m\u001b[43murl\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    115\u001b[0m \u001b[43m        \u001b[49m\u001b[43mparams\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mparams\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    116\u001b[0m \u001b[43m        \u001b[49m\u001b[43msupplied_headers\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mheaders\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    117\u001b[0m \u001b[43m        \u001b[49m\u001b[43mfiles\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mfiles\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    118\u001b[0m \u001b[43m        \u001b[49m\u001b[43mstream\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mstream\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    119\u001b[0m \u001b[43m        \u001b[49m\u001b[43mrequest_id\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mrequest_id\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    120\u001b[0m \u001b[43m    \u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    121\u001b[0m     resp, got_stream \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_interpret_response(result, stream)\n\u001b[1;32m    122\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m resp, got_stream, \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mapi_key\n",
      "File \u001b[0;32m~/miniconda3/envs/dl_py3.8/lib/python3.8/site-packages/openai/api_requestor.py:301\u001b[0m, in \u001b[0;36mAPIRequestor.request_raw\u001b[0;34m(self, method, url, params, supplied_headers, files, stream, request_id)\u001b[0m\n\u001b[1;32m    291\u001b[0m     result \u001b[38;5;241m=\u001b[39m _thread_context\u001b[38;5;241m.\u001b[39msession\u001b[38;5;241m.\u001b[39mrequest(\n\u001b[1;32m    292\u001b[0m         method,\n\u001b[1;32m    293\u001b[0m         abs_url,\n\u001b[0;32m   (...)\u001b[0m\n\u001b[1;32m    298\u001b[0m         timeout\u001b[38;5;241m=\u001b[39mTIMEOUT_SECS,\n\u001b[1;32m    299\u001b[0m     )\n\u001b[1;32m    300\u001b[0m \u001b[38;5;28;01mexcept\u001b[39;00m requests\u001b[38;5;241m.\u001b[39mexceptions\u001b[38;5;241m.\u001b[39mRequestException \u001b[38;5;28;01mas\u001b[39;00m e:\n\u001b[0;32m--> 301\u001b[0m     \u001b[38;5;28;01mraise\u001b[39;00m error\u001b[38;5;241m.\u001b[39mAPIConnectionError(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mError communicating with OpenAI\u001b[39m\u001b[38;5;124m\"\u001b[39m) \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01me\u001b[39;00m\n\u001b[1;32m    302\u001b[0m util\u001b[38;5;241m.\u001b[39mlog_info(\n\u001b[1;32m    303\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mOpenAI API response\u001b[39m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m    304\u001b[0m     path\u001b[38;5;241m=\u001b[39mabs_url,\n\u001b[1;32m    305\u001b[0m     response_code\u001b[38;5;241m=\u001b[39mresult\u001b[38;5;241m.\u001b[39mstatus_code,\n\u001b[1;32m    306\u001b[0m     processing_ms\u001b[38;5;241m=\u001b[39mresult\u001b[38;5;241m.\u001b[39mheaders\u001b[38;5;241m.\u001b[39mget(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mOpenAI-Processing-Ms\u001b[39m\u001b[38;5;124m\"\u001b[39m),\n\u001b[1;32m    307\u001b[0m )\n\u001b[1;32m    308\u001b[0m \u001b[38;5;66;03m# Don't read the whole stream for debug logging unless necessary.\u001b[39;00m\n",
      "\u001b[0;31mAPIConnectionError\u001b[0m: Error communicating with OpenAI"
     ]
    }
   ],
   "source": [
    "q_total = []\n",
    "ans_total = []\n",
    "ans_gold_total = []\n",
    "i = 0\n",
    "    \n",
    "for case in tqdm(dev_data_200):\n",
    "    if(i < 98): \n",
    "        i += 1\n",
    "        continue\n",
    "    q = case['Problem']\n",
    "    c = case['options']\n",
    "    a = case['Rationale']\n",
    "    cr = case['correct']\n",
    "\n",
    "    ans_dict = get_ans_dict(c)\n",
    "\n",
    "    a += ' ' + ans_dict[cr]\n",
    "\n",
    "    prompt_q = '' \n",
    "    prompt_q += '\\nQuestion: ' + q + '\\n'\n",
    "    prompt_q += 'Options: ' + c + '\\n'\n",
    "    prompt_q += \"Let's think step by step\\n\"\n",
    "\n",
    "    prompt_full = ''.join(prompt_complex) + prompt_q\n",
    "    response = openai.Completion.create(model=\"text-davinci-002\", \n",
    "                                        prompt=prompt_full, \n",
    "                                        temperature=0.5, \n",
    "                                        max_tokens=256,\n",
    "                                        n=50\n",
    "                                       )\n",
    "        \n",
    "    ans_strs = []\n",
    "    for ans in response['choices']: ans_strs.append(ans['text'])\n",
    "    ans_total.append(ans_strs)\n",
    "    q_total.append(q)\n",
    "    ans_gold_total.append(a)\n",
    "    i += 1\n",
    "    time.sleep(10)\n",
    "    # if(i == 2): break"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 85,
   "id": "3ca30410-c74b-4cd2-8976-557892f49841",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Correct 49 total 101 acc 0.4851\n"
     ]
    }
   ],
   "source": [
    "total, acc = 0, 0\n",
    "\n",
    "with open('outputs/dev_consistency_complex_from_98.txt', 'w') as fd:\n",
    "    for q, a, pred in zip(q_total, ans_gold_total, ans_total):\n",
    "        fd.write('Q: ' + q + ' [EOQ]\\n--\\n')\n",
    "        fd.write('A_model:\\n')\n",
    "        for p in pred:\n",
    "            fd.write(p + ' [EOA]\\n\\n')\n",
    "        majority = find_majority(pred)\n",
    "        fd.write('\\nMajority = ' + majority + '\\n')\n",
    "        fd.write('--\\n')\n",
    "        fd.write('A_gold:\\n' + a + '\\n')\n",
    "        fd.write('\\n\\n----\\n\\n')\n",
    "        \n",
    "        if(test_answer(majority, a)):\n",
    "            acc += 1\n",
    "        total += 1\n",
    "\n",
    "print('Correct %d total %d acc %.4f' % (acc, total, float(acc) / total))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 96,
   "id": "16692cd6-af40-4c5b-8d4b-40aaeb9ba99c",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 86,
   "id": "8aeaaef0-4057-4ee9-ad9a-3f866c3adf9d",
   "metadata": {},
   "outputs": [],
   "source": [
    "questions, answers, answer_reasons, answer_labels = read_sample_answer(\"outputs/dev_consistency_complex.txt\")\n",
    "questions_, answers_, answer_reasons_, answer_labels_ = read_sample_answer(\"outputs/dev_consistency_complex_from_98.txt\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 87,
   "id": "d0366e1a-344a-457f-bf05-0c6523a8fb5e",
   "metadata": {},
   "outputs": [],
   "source": [
    "answers.extend(answers_)\n",
    "answer_labels.extend(answer_labels_)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 88,
   "id": "e6e2ee16-e596-436b-bd72-8c3bd67dc585",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "199 total 76 correct acc 0.3819\n"
     ]
    }
   ],
   "source": [
    "test_voting_over_k(10, answers, answer_labels)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 89,
   "id": "47ae3afa-0a66-41f5-a363-80bf802dc574",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "199 total 92 correct acc 0.4623\n"
     ]
    }
   ],
   "source": [
    "test_voting_over_k(20, answers, answer_labels)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 90,
   "id": "1d2a48ef-76ba-4713-a9bb-d0204b9e1bd6",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "199 total 95 correct acc 0.4774\n"
     ]
    }
   ],
   "source": [
    "test_voting_over_k(30, answers, answer_labels)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 106,
   "id": "c2174cda-10a6-4728-a01d-90a6757f7d3b",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "199 total 96 correct acc 0.4824\n"
     ]
    }
   ],
   "source": [
    "test_voting_over_k(35, answers, answer_labels)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 91,
   "id": "dbdcbaf6-85fc-46c4-ad3f-28919332e952",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "199 total 93 correct acc 0.4673\n"
     ]
    }
   ],
   "source": [
    "test_voting_over_k(40, answers, answer_labels)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 92,
   "id": "2c7c606c-61f6-489d-a551-1fed25f6744e",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "199 total 97 correct acc 0.4874\n"
     ]
    }
   ],
   "source": [
    "test_voting_over_k(50, answers, answer_labels)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 97,
   "id": "99fbf8b9-5615-4725-a72c-92f9aaa631c6",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "199 total 97 correct acc 0.4874\n"
     ]
    }
   ],
   "source": [
    "test_voting_over_hard_and_long(50, answers, answer_labels)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 98,
   "id": "1933c070-7a64-4eb2-9ebb-9716c7b2eb7a",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "199 total 97 correct acc 0.4874\n"
     ]
    }
   ],
   "source": [
    "test_voting_over_hard_and_long(40, answers, answer_labels)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 99,
   "id": "9bf1a602-2801-4cb3-8b4c-91dc619f9a5f",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "199 total 93 correct acc 0.4673\n"
     ]
    }
   ],
   "source": [
    "test_voting_over_hard_and_long(30, answers, answer_labels)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 100,
   "id": "0b3e5c91-a17b-4ec1-bada-7d95d105f89f",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "199 total 92 correct acc 0.4623\n"
     ]
    }
   ],
   "source": [
    "test_voting_over_hard_and_long(20, answers, answer_labels)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 101,
   "id": "cdc5adba-f0be-41d3-bb66-550fb0b7d6b0",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "199 total 75 correct acc 0.3769\n"
     ]
    }
   ],
   "source": [
    "test_voting_over_hard_and_long(10, answers, answer_labels)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "95e34219-222f-40b9-a223-d79319fde7d9",
   "metadata": {},
   "source": [
    "# Long formula Prompt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 112,
   "id": "dfd45ac4-340d-406e-bc94-b11310aba9af",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 200/200 [1:08:33<00:00, 20.57s/it]\n"
     ]
    }
   ],
   "source": [
    "q_total = []\n",
    "ans_total = []\n",
    "ans_gold_total = []\n",
    "i = 0\n",
    "    \n",
    "for case in tqdm(dev_data_200):\n",
    "    q = case['Problem']\n",
    "    c = case['options']\n",
    "    a = case['Rationale']\n",
    "    cr = case['correct']\n",
    "\n",
    "    ans_dict = get_ans_dict(c)\n",
    "\n",
    "    a += ' ' + ans_dict[cr]\n",
    "\n",
    "    prompt_q = '' \n",
    "    prompt_q += '\\nQuestion: ' + q + '\\n'\n",
    "    prompt_q += 'Options: ' + c + '\\n'\n",
    "    prompt_q += \"Let's think step by step\\n\"\n",
    "\n",
    "    prompt_full = ''.join(prompt_long_formula) + prompt_q\n",
    "    response = openai.Completion.create(model=\"text-davinci-002\", \n",
    "                                        prompt=prompt_full, \n",
    "                                        temperature=0.5, \n",
    "                                        max_tokens=256,\n",
    "                                        n=50\n",
    "                                       )\n",
    "        \n",
    "    ans_strs = []\n",
    "    for ans in response['choices']: ans_strs.append(ans['text'])\n",
    "    ans_total.append(ans_strs)\n",
    "    q_total.append(q)\n",
    "    ans_gold_total.append(a)\n",
    "    i += 1\n",
    "    time.sleep(10)\n",
    "    # if(i == 2): break"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 113,
   "id": "86e2e692-3f03-426b-b775-10d48032f17e",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Correct 101 total 200 acc 0.5050\n"
     ]
    }
   ],
   "source": [
    "total, acc = 0, 0\n",
    "\n",
    "with open('outputs/dev_consistency_long_formula.txt', 'w') as fd:\n",
    "    for q, a, pred in zip(q_total, ans_gold_total, ans_total):\n",
    "        fd.write('Q: ' + q + ' [EOQ]\\n--\\n')\n",
    "        fd.write('A_model:\\n')\n",
    "        for p in pred:\n",
    "            fd.write(p + ' [EOA]\\n\\n')\n",
    "        majority = find_majority(pred)\n",
    "        fd.write('\\nMajority = ' + majority + '\\n')\n",
    "        fd.write('--\\n')\n",
    "        fd.write('A_gold:\\n' + a + '\\n')\n",
    "        fd.write('\\n\\n----\\n\\n')\n",
    "        \n",
    "        if(test_answer(majority, a)):\n",
    "            acc += 1\n",
    "        total += 1\n",
    "\n",
    "print('Correct %d total %d acc %.4f' % (acc, total, float(acc) / total))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 161,
   "id": "6147fe06-751f-4e59-a12d-6fee3f9ec676",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "k = 10\n",
      "200 total 77 correct acc 0.3850\n",
      "k = 20\n",
      "200 total 92 correct acc 0.4600\n",
      "k = 30\n",
      "200 total 97 correct acc 0.4850\n",
      "k = 40\n",
      "200 total 103 correct acc 0.5150\n"
     ]
    }
   ],
   "source": [
    "questions, answers, answer_reasons, answer_labels = read_sample_answer(\"outputs/dev_consistency_long_formula.txt\")\n",
    "for k in [10, 20, 30, 40]:\n",
    "    print('k = %d' % k)\n",
    "    test_voting_over_k(k, answers, answer_labels)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 162,
   "id": "f0c75483-07f9-438b-8115-2a595cb9b7b9",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "k = 10\n",
      "200 total 85 correct acc 0.4250\n",
      "k = 20\n",
      "200 total 97 correct acc 0.4850\n",
      "k = 30\n",
      "200 total 98 correct acc 0.4900\n",
      "k = 40\n",
      "200 total 98 correct acc 0.4900\n"
     ]
    }
   ],
   "source": [
    "for k in [10, 20, 30, 40]:\n",
    "    print('k = %d' % k)\n",
    "    test_voting_over_k(k, answers, answer_labels, vote_over_easy=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "1d54ffca-ab1d-48ee-ac94-6f1cadff7a6b",
   "metadata": {},
   "source": [
    "# Test set"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "0982156f-75a8-45d5-98f6-bc62ced5f00b",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "id": "57e309b3-092c-4bb0-bd4b-22157d496bbb",
   "metadata": {},
   "source": [
    "## Original CoT"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 139,
   "id": "f42878cc-a6bc-4733-8cd6-35d3f33f78d9",
   "metadata": {
    "collapsed": true,
    "jupyter": {
     "outputs_hidden": true
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 26%|████████████████████████████████▊                                                                                               | 154/600 [51:59<2:30:33, 20.25s/it]\n"
     ]
    },
    {
     "ename": "RateLimitError",
     "evalue": "Rate limit reached for default-text-davinci-002 in organization org-w7nKit9OwsqNNO3i9GmXe5uk on tokens per min. Limit: 250000.000000 / min. Current: 312060.000000 / min. Contact support@openai.com if you continue to have issues.",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mRateLimitError\u001b[0m                            Traceback (most recent call last)",
      "Input \u001b[0;32mIn [139]\u001b[0m, in \u001b[0;36m<cell line: 7>\u001b[0;34m()\u001b[0m\n\u001b[1;32m     20\u001b[0m prompt_q \u001b[38;5;241m+\u001b[39m\u001b[38;5;241m=\u001b[39m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mLet\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124ms think step by step\u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m\n\u001b[1;32m     22\u001b[0m prompt_full \u001b[38;5;241m=\u001b[39m \u001b[38;5;124m'\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;241m.\u001b[39mjoin(\u001b[38;5;28meval\u001b[39m(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mprompt_\u001b[39m\u001b[38;5;124m\"\u001b[39m \u001b[38;5;241m+\u001b[39m prompt_type)) \u001b[38;5;241m+\u001b[39m prompt_q\n\u001b[0;32m---> 23\u001b[0m response \u001b[38;5;241m=\u001b[39m \u001b[43mopenai\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mCompletion\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mcreate\u001b[49m\u001b[43m(\u001b[49m\u001b[43mmodel\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mtext-davinci-002\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\n\u001b[1;32m     24\u001b[0m \u001b[43m                                    \u001b[49m\u001b[43mprompt\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mprompt_full\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\n\u001b[1;32m     25\u001b[0m \u001b[43m                                    \u001b[49m\u001b[43mtemperature\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;241;43m0.5\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\n\u001b[1;32m     26\u001b[0m \u001b[43m                                    \u001b[49m\u001b[43mmax_tokens\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;241;43m256\u001b[39;49m\u001b[43m,\u001b[49m\n\u001b[1;32m     27\u001b[0m \u001b[43m                                    \u001b[49m\u001b[43mn\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;241;43m50\u001b[39;49m\n\u001b[1;32m     28\u001b[0m \u001b[43m                                   \u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m     30\u001b[0m ans_strs \u001b[38;5;241m=\u001b[39m []\n\u001b[1;32m     31\u001b[0m \u001b[38;5;28;01mfor\u001b[39;00m ans \u001b[38;5;129;01min\u001b[39;00m response[\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mchoices\u001b[39m\u001b[38;5;124m'\u001b[39m]: ans_strs\u001b[38;5;241m.\u001b[39mappend(ans[\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mtext\u001b[39m\u001b[38;5;124m'\u001b[39m])\n",
      "File \u001b[0;32m~/miniconda3/envs/dl_py3.8/lib/python3.8/site-packages/openai/api_resources/completion.py:25\u001b[0m, in \u001b[0;36mCompletion.create\u001b[0;34m(cls, *args, **kwargs)\u001b[0m\n\u001b[1;32m     23\u001b[0m \u001b[38;5;28;01mwhile\u001b[39;00m \u001b[38;5;28;01mTrue\u001b[39;00m:\n\u001b[1;32m     24\u001b[0m     \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[0;32m---> 25\u001b[0m         \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43msuper\u001b[39;49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mcreate\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m     26\u001b[0m     \u001b[38;5;28;01mexcept\u001b[39;00m TryAgain \u001b[38;5;28;01mas\u001b[39;00m e:\n\u001b[1;32m     27\u001b[0m         \u001b[38;5;28;01mif\u001b[39;00m timeout \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m \u001b[38;5;129;01mand\u001b[39;00m time\u001b[38;5;241m.\u001b[39mtime() \u001b[38;5;241m>\u001b[39m start \u001b[38;5;241m+\u001b[39m timeout:\n",
      "File \u001b[0;32m~/miniconda3/envs/dl_py3.8/lib/python3.8/site-packages/openai/api_resources/abstract/engine_api_resource.py:115\u001b[0m, in \u001b[0;36mEngineAPIResource.create\u001b[0;34m(cls, api_key, api_base, api_type, request_id, api_version, organization, **params)\u001b[0m\n\u001b[1;32m    107\u001b[0m requestor \u001b[38;5;241m=\u001b[39m api_requestor\u001b[38;5;241m.\u001b[39mAPIRequestor(\n\u001b[1;32m    108\u001b[0m     api_key,\n\u001b[1;32m    109\u001b[0m     api_base\u001b[38;5;241m=\u001b[39mapi_base,\n\u001b[0;32m   (...)\u001b[0m\n\u001b[1;32m    112\u001b[0m     organization\u001b[38;5;241m=\u001b[39morganization,\n\u001b[1;32m    113\u001b[0m )\n\u001b[1;32m    114\u001b[0m url \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mcls\u001b[39m\u001b[38;5;241m.\u001b[39mclass_url(engine, api_type, api_version)\n\u001b[0;32m--> 115\u001b[0m response, _, api_key \u001b[38;5;241m=\u001b[39m \u001b[43mrequestor\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mrequest\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m    116\u001b[0m \u001b[43m    \u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mpost\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m,\u001b[49m\n\u001b[1;32m    117\u001b[0m \u001b[43m    \u001b[49m\u001b[43murl\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    118\u001b[0m \u001b[43m    \u001b[49m\u001b[43mparams\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mparams\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    119\u001b[0m \u001b[43m    \u001b[49m\u001b[43mheaders\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mheaders\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    120\u001b[0m \u001b[43m    \u001b[49m\u001b[43mstream\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mstream\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    121\u001b[0m \u001b[43m    \u001b[49m\u001b[43mrequest_id\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mrequest_id\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    122\u001b[0m \u001b[43m\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    124\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m stream:\n\u001b[1;32m    125\u001b[0m     \u001b[38;5;66;03m# must be an iterator\u001b[39;00m\n\u001b[1;32m    126\u001b[0m     \u001b[38;5;28;01massert\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;28misinstance\u001b[39m(response, OpenAIResponse)\n",
      "File \u001b[0;32m~/miniconda3/envs/dl_py3.8/lib/python3.8/site-packages/openai/api_requestor.py:121\u001b[0m, in \u001b[0;36mAPIRequestor.request\u001b[0;34m(self, method, url, params, headers, files, stream, request_id)\u001b[0m\n\u001b[1;32m    102\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mrequest\u001b[39m(\n\u001b[1;32m    103\u001b[0m     \u001b[38;5;28mself\u001b[39m,\n\u001b[1;32m    104\u001b[0m     method,\n\u001b[0;32m   (...)\u001b[0m\n\u001b[1;32m    110\u001b[0m     request_id: Optional[\u001b[38;5;28mstr\u001b[39m] \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mNone\u001b[39;00m,\n\u001b[1;32m    111\u001b[0m ) \u001b[38;5;241m-\u001b[39m\u001b[38;5;241m>\u001b[39m Tuple[Union[OpenAIResponse, Iterator[OpenAIResponse]], \u001b[38;5;28mbool\u001b[39m, \u001b[38;5;28mstr\u001b[39m]:\n\u001b[1;32m    112\u001b[0m     result \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mrequest_raw(\n\u001b[1;32m    113\u001b[0m         method\u001b[38;5;241m.\u001b[39mlower(),\n\u001b[1;32m    114\u001b[0m         url,\n\u001b[0;32m   (...)\u001b[0m\n\u001b[1;32m    119\u001b[0m         request_id\u001b[38;5;241m=\u001b[39mrequest_id,\n\u001b[1;32m    120\u001b[0m     )\n\u001b[0;32m--> 121\u001b[0m     resp, got_stream \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_interpret_response\u001b[49m\u001b[43m(\u001b[49m\u001b[43mresult\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mstream\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    122\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m resp, got_stream, \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mapi_key\n",
      "File \u001b[0;32m~/miniconda3/envs/dl_py3.8/lib/python3.8/site-packages/openai/api_requestor.py:328\u001b[0m, in \u001b[0;36mAPIRequestor._interpret_response\u001b[0;34m(self, result, stream)\u001b[0m\n\u001b[1;32m    320\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m (\n\u001b[1;32m    321\u001b[0m         \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_interpret_response_line(\n\u001b[1;32m    322\u001b[0m             line, result\u001b[38;5;241m.\u001b[39mstatus_code, result\u001b[38;5;241m.\u001b[39mheaders, stream\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mTrue\u001b[39;00m\n\u001b[1;32m    323\u001b[0m         )\n\u001b[1;32m    324\u001b[0m         \u001b[38;5;28;01mfor\u001b[39;00m line \u001b[38;5;129;01min\u001b[39;00m parse_stream(result\u001b[38;5;241m.\u001b[39miter_lines())\n\u001b[1;32m    325\u001b[0m     ), \u001b[38;5;28;01mTrue\u001b[39;00m\n\u001b[1;32m    326\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[1;32m    327\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m (\n\u001b[0;32m--> 328\u001b[0m         \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_interpret_response_line\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m    329\u001b[0m \u001b[43m            \u001b[49m\u001b[43mresult\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mcontent\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mresult\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mstatus_code\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mresult\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mheaders\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mstream\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;28;43;01mFalse\u001b[39;49;00m\n\u001b[1;32m    330\u001b[0m \u001b[43m        \u001b[49m\u001b[43m)\u001b[49m,\n\u001b[1;32m    331\u001b[0m         \u001b[38;5;28;01mFalse\u001b[39;00m,\n\u001b[1;32m    332\u001b[0m     )\n",
      "File \u001b[0;32m~/miniconda3/envs/dl_py3.8/lib/python3.8/site-packages/openai/api_requestor.py:361\u001b[0m, in \u001b[0;36mAPIRequestor._interpret_response_line\u001b[0;34m(self, rbody, rcode, rheaders, stream)\u001b[0m\n\u001b[1;32m    359\u001b[0m stream_error \u001b[38;5;241m=\u001b[39m stream \u001b[38;5;129;01mand\u001b[39;00m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124merror\u001b[39m\u001b[38;5;124m\"\u001b[39m \u001b[38;5;129;01min\u001b[39;00m resp\u001b[38;5;241m.\u001b[39mdata\n\u001b[1;32m    360\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m stream_error \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;241m200\u001b[39m \u001b[38;5;241m<\u001b[39m\u001b[38;5;241m=\u001b[39m rcode \u001b[38;5;241m<\u001b[39m \u001b[38;5;241m300\u001b[39m:\n\u001b[0;32m--> 361\u001b[0m     \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mhandle_error_response(\n\u001b[1;32m    362\u001b[0m         rbody, rcode, resp\u001b[38;5;241m.\u001b[39mdata, rheaders, stream_error\u001b[38;5;241m=\u001b[39mstream_error\n\u001b[1;32m    363\u001b[0m     )\n\u001b[1;32m    364\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m resp\n",
      "\u001b[0;31mRateLimitError\u001b[0m: Rate limit reached for default-text-davinci-002 in organization org-w7nKit9OwsqNNO3i9GmXe5uk on tokens per min. Limit: 250000.000000 / min. Current: 312060.000000 / min. Contact support@openai.com if you continue to have issues."
     ]
    }
   ],
   "source": [
    "prompt_type = \"original\"\n",
    "\n",
    "q_total = []\n",
    "ans_total = []\n",
    "ans_gold_total = []\n",
    "i = 0\n",
    "for case in tqdm(test_data):\n",
    "    q = case['Problem']\n",
    "    c = case['options']\n",
    "    a = case['Rationale']\n",
    "    cr = case['correct']\n",
    "\n",
    "    ans_dict = get_ans_dict(c)\n",
    "\n",
    "    a += ' ' + ans_dict[cr]\n",
    "\n",
    "    prompt_q = '' \n",
    "    prompt_q += '\\nQuestion: ' + q + '\\n'\n",
    "    prompt_q += 'Options: ' + c + '\\n'\n",
    "    prompt_q += \"Let's think step by step\\n\"\n",
    "\n",
    "    prompt_full = ''.join(eval(\"prompt_\" + prompt_type)) + prompt_q\n",
    "    response = openai.Completion.create(model=\"text-davinci-002\", \n",
    "                                        prompt=prompt_full, \n",
    "                                        temperature=0.5, \n",
    "                                        max_tokens=256,\n",
    "                                        n=50\n",
    "                                       )\n",
    "        \n",
    "    ans_strs = []\n",
    "    for ans in response['choices']: ans_strs.append(ans['text'])\n",
    "    ans_total.append(ans_strs)\n",
    "    q_total.append(q)\n",
    "    ans_gold_total.append(a)\n",
    "    i += 1\n",
    "    time.sleep(10)\n",
    "    # if(i == 2): break"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 154,
   "id": "7017eb72-e4c0-4dd2-b23c-5954e2fd450b",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Correct 64 total 154 acc 0.4156\n"
     ]
    }
   ],
   "source": [
    "total, acc = 0, 0\n",
    "with open('outputs/test_consistency_%s.txt' % prompt_type, 'w') as fd:\n",
    "    for q, a, pred in zip(q_total, ans_gold_total, ans_total):\n",
    "        fd.write('Q: ' + q + ' [EOQ]\\n--\\n')\n",
    "        fd.write('A_model:\\n')\n",
    "        for p in pred:\n",
    "            fd.write(p + ' [EOA]\\n\\n')\n",
    "        majority = find_majority(pred)\n",
    "        fd.write('\\nMajority = ' + majority + '\\n')\n",
    "        fd.write('--\\n')\n",
    "        fd.write('A_gold:\\n' + a + '\\n')\n",
    "        fd.write('\\n\\n----\\n\\n')\n",
    "        \n",
    "        if(test_answer(majority, a)):\n",
    "            acc += 1\n",
    "        total += 1\n",
    "\n",
    "print('Correct %d total %d acc %.4f' % (acc, total, float(acc) / total))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 155,
   "id": "a5b0a9c7-731e-44fb-8c9a-99fd25f07634",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 600/600 [2:28:13<00:00, 14.82s/it]\n"
     ]
    }
   ],
   "source": [
    "prompt_type = \"original\"\n",
    "\n",
    "q_total = []\n",
    "ans_total = []\n",
    "ans_gold_total = []\n",
    "i = 0\n",
    "for case in tqdm(test_data):\n",
    "    if(i < 154):\n",
    "        i += 1\n",
    "        continue\n",
    "    q = case['Problem']\n",
    "    c = case['options']\n",
    "    a = case['Rationale']\n",
    "    cr = case['correct']\n",
    "\n",
    "    ans_dict = get_ans_dict(c)\n",
    "\n",
    "    a += ' ' + ans_dict[cr]\n",
    "\n",
    "    prompt_q = '' \n",
    "    prompt_q += '\\nQuestion: ' + q + '\\n'\n",
    "    prompt_q += 'Options: ' + c + '\\n'\n",
    "    prompt_q += \"Let's think step by step\\n\"\n",
    "\n",
    "    prompt_full = ''.join(eval(\"prompt_\" + prompt_type)) + prompt_q\n",
    "    response = openai.Completion.create(model=\"text-davinci-002\", \n",
    "                                        prompt=prompt_full, \n",
    "                                        temperature=0.5, \n",
    "                                        max_tokens=256,\n",
    "                                        n=50\n",
    "                                       )\n",
    "        \n",
    "    ans_strs = []\n",
    "    for ans in response['choices']: ans_strs.append(ans['text'])\n",
    "    ans_total.append(ans_strs)\n",
    "    q_total.append(q)\n",
    "    ans_gold_total.append(a)\n",
    "    i += 1\n",
    "    time.sleep(10)\n",
    "    # if(i == 2): break"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 156,
   "id": "e02659bf-87ed-4586-8e3b-5c6a44195faa",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Correct 199 total 446 acc 0.4462\n"
     ]
    }
   ],
   "source": [
    "total, acc = 0, 0\n",
    "with open('outputs/test_consistency_%s_from_154.txt' % prompt_type, 'w') as fd:\n",
    "    for q, a, pred in zip(q_total, ans_gold_total, ans_total):\n",
    "        fd.write('Q: ' + q + ' [EOQ]\\n--\\n')\n",
    "        fd.write('A_model:\\n')\n",
    "        for p in pred:\n",
    "            fd.write(p + ' [EOA]\\n\\n')\n",
    "        majority = find_majority(pred)\n",
    "        fd.write('\\nMajority = ' + majority + '\\n')\n",
    "        fd.write('--\\n')\n",
    "        fd.write('A_gold:\\n' + a + '\\n')\n",
    "        fd.write('\\n\\n----\\n\\n')\n",
    "        \n",
    "        if(test_answer(majority, a)):\n",
    "            acc += 1\n",
    "        total += 1\n",
    "\n",
    "print('Correct %d total %d acc %.4f' % (acc, total, float(acc) / total))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 157,
   "id": "ea09ac68-e07a-40e5-aefb-450bf0d4f117",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.43833333333333335"
      ]
     },
     "execution_count": 157,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "(199 + 64) / 600"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "3fe8f8fb-6843-4ac3-b09f-8bb48e472196",
   "metadata": {},
   "source": [
    "## Random CoT"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 163,
   "id": "0b225e77-1122-4a44-8334-40d07a9b0300",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|███████████████████████████████████████████████████████████████████████████████████████████████████████████| 600/600 [3:39:57<00:00, 22.00s/it]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Correct 291 total 600 acc 0.4850\n"
     ]
    }
   ],
   "source": [
    "prompt_type = \"random\"\n",
    "\n",
    "q_total = []\n",
    "ans_total = []\n",
    "ans_gold_total = []\n",
    "i = 0\n",
    "for case in tqdm(test_data):\n",
    "    q = case['Problem']\n",
    "    c = case['options']\n",
    "    a = case['Rationale']\n",
    "    cr = case['correct']\n",
    "\n",
    "    ans_dict = get_ans_dict(c)\n",
    "\n",
    "    a += ' ' + ans_dict[cr]\n",
    "\n",
    "    prompt_q = '' \n",
    "    prompt_q += '\\nQuestion: ' + q + '\\n'\n",
    "    prompt_q += 'Options: ' + c + '\\n'\n",
    "    prompt_q += \"Let's think step by step\\n\"\n",
    "\n",
    "    prompt_full = ''.join(eval(\"prompt_\" + prompt_type)) + prompt_q\n",
    "    response = openai.Completion.create(model=\"text-davinci-002\", \n",
    "                                        prompt=prompt_full, \n",
    "                                        temperature=0.5, \n",
    "                                        max_tokens=256,\n",
    "                                        n=50\n",
    "                                       )\n",
    "        \n",
    "    ans_strs = []\n",
    "    for ans in response['choices']: ans_strs.append(ans['text'])\n",
    "    ans_total.append(ans_strs)\n",
    "    q_total.append(q)\n",
    "    ans_gold_total.append(a)\n",
    "    i += 1\n",
    "    time.sleep(10)\n",
    "    # if(i == 2): break\n",
    "    \n",
    "total, acc = 0, 0\n",
    "with open('outputs/test_consistency_%s.txt' % prompt_type, 'w') as fd:\n",
    "    for q, a, pred in zip(q_total, ans_gold_total, ans_total):\n",
    "        fd.write('Q: ' + q + ' [EOQ]\\n--\\n')\n",
    "        fd.write('A_model:\\n')\n",
    "        for p in pred:\n",
    "            fd.write(p + ' [EOA]\\n\\n')\n",
    "        majority = find_majority(pred)\n",
    "        fd.write('\\nMajority = ' + majority + '\\n')\n",
    "        fd.write('--\\n')\n",
    "        fd.write('A_gold:\\n' + a + '\\n')\n",
    "        fd.write('\\n\\n----\\n\\n')\n",
    "        \n",
    "        if(test_answer(majority, a)):\n",
    "            acc += 1\n",
    "        total += 1\n",
    "\n",
    "print('Correct %d total %d acc %.4f' % (acc, total, float(acc) / total))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "2ed88fb9-c4e0-455a-91fd-0129f68014de",
   "metadata": {},
   "source": [
    "## Complex CoT"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 158,
   "id": "34b2fbe3-92d5-492c-8d9a-da86d5c6b0b9",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|███████████████████████████████████████████████████████████████████████████████████████████████████████████| 600/600 [3:25:53<00:00, 20.59s/it]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Correct 297 total 600 acc 0.4950\n"
     ]
    }
   ],
   "source": [
    "prompt_type = \"long_formula\"\n",
    "\n",
    "q_total = []\n",
    "ans_total = []\n",
    "ans_gold_total = []\n",
    "i = 0\n",
    "for case in tqdm(test_data):\n",
    "    q = case['Problem']\n",
    "    c = case['options']\n",
    "    a = case['Rationale']\n",
    "    cr = case['correct']\n",
    "\n",
    "    ans_dict = get_ans_dict(c)\n",
    "\n",
    "    a += ' ' + ans_dict[cr]\n",
    "\n",
    "    prompt_q = '' \n",
    "    prompt_q += '\\nQuestion: ' + q + '\\n'\n",
    "    prompt_q += 'Options: ' + c + '\\n'\n",
    "    prompt_q += \"Let's think step by step\\n\"\n",
    "\n",
    "    prompt_full = ''.join(eval(\"prompt_\" + prompt_type)) + prompt_q\n",
    "    response = openai.Completion.create(model=\"text-davinci-002\", \n",
    "                                        prompt=prompt_full, \n",
    "                                        temperature=0.5, \n",
    "                                        max_tokens=256,\n",
    "                                        n=50\n",
    "                                       )\n",
    "        \n",
    "    ans_strs = []\n",
    "    for ans in response['choices']: ans_strs.append(ans['text'])\n",
    "    ans_total.append(ans_strs)\n",
    "    q_total.append(q)\n",
    "    ans_gold_total.append(a)\n",
    "    i += 1\n",
    "    time.sleep(10)\n",
    "    # if(i == 2): break\n",
    "    \n",
    "total, acc = 0, 0\n",
    "with open('outputs/test_consistency_%s.txt' % prompt_type, 'w') as fd:\n",
    "    for q, a, pred in zip(q_total, ans_gold_total, ans_total):\n",
    "        fd.write('Q: ' + q + ' [EOQ]\\n--\\n')\n",
    "        fd.write('A_model:\\n')\n",
    "        for p in pred:\n",
    "            fd.write(p + ' [EOA]\\n\\n')\n",
    "        majority = find_majority(pred)\n",
    "        fd.write('\\nMajority = ' + majority + '\\n')\n",
    "        fd.write('--\\n')\n",
    "        fd.write('A_gold:\\n' + a + '\\n')\n",
    "        fd.write('\\n\\n----\\n\\n')\n",
    "        \n",
    "        if(test_answer(majority, a)):\n",
    "            acc += 1\n",
    "        total += 1\n",
    "\n",
    "print('Correct %d total %d acc %.4f' % (acc, total, float(acc) / total))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 160,
   "id": "aae9356c-cd1b-4b73-914b-ca22e7d36251",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "600 total 301 correct acc 0.5017\n"
     ]
    }
   ],
   "source": [
    "questions, answers, answer_reasons, answer_labels = read_sample_answer(\"outputs/test_consistency_long_formula.txt\")\n",
    "test_voting_over_k(40, answers, answer_labels)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
